From 133b13a7a68656d367d84723f303514af0026fb9 Mon Sep 17 00:00:00 2001 From: Manuel Schiller Date: Wed, 3 Jun 2026 08:38:35 +0200 Subject: [PATCH 1/2] Implement operator-managed runtime architecture --- .github/kind-runtime-config.yaml | 11 + .github/workflows/ci.yml | 171 ++++-- .gitignore | 1 + Dockerfile | 6 +- Dockerfile.runtime | 35 ++ Makefile | 31 +- README.md | 114 ++-- api/v1alpha1/smolvm_types.go | 4 + api/v1alpha1/smolvmnode_types.go | 125 +++++ api/v1alpha1/zz_generated.deepcopy.go | 123 +++++ cmd/runtime-agent/main.go | 500 ++++++++++++++++++ .../crd/bases/vm.smolvm.dev_smolvmnodes.yaml | 168 ++++++ config/crd/bases/vm.smolvm.dev_smolvms.yaml | 6 + config/crd/kustomization.yaml | 1 + config/default/kustomization.yaml | 1 + config/default/manager_auth_proxy_patch.yaml | 2 +- config/default/manager_config_patch.yaml | 2 +- config/manager/manager.yaml | 60 +-- config/rbac/role.yaml | 28 +- config/runtime/daemonset.yaml | 97 ++++ config/runtime/kustomization.yaml | 8 + config/runtime/networkpolicy.yaml | 20 + config/samples/vm_v1alpha1_smolvm.yaml | 2 - internal/controller/smolvm_controller.go | 391 ++++++++++++-- internal/controller/smolvm_controller_test.go | 286 +++++++++- internal/smolvm/client.go | 71 +++ test/e2e/e2e_test.go | 29 +- 27 files changed, 2067 insertions(+), 226 deletions(-) create mode 100644 .github/kind-runtime-config.yaml create mode 100644 Dockerfile.runtime create mode 100644 api/v1alpha1/smolvmnode_types.go create mode 100644 cmd/runtime-agent/main.go create mode 100644 config/crd/bases/vm.smolvm.dev_smolvmnodes.yaml create mode 100644 config/runtime/daemonset.yaml create mode 100644 config/runtime/kustomization.yaml create mode 100644 config/runtime/networkpolicy.yaml diff --git a/.github/kind-runtime-config.yaml b/.github/kind-runtime-config.yaml new file mode 100644 index 0000000..5b5bb3c --- /dev/null +++ b/.github/kind-runtime-config.yaml @@ -0,0 +1,11 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane + extraMounts: + - hostPath: /dev/kvm + containerPath: /dev/kvm + - hostPath: /tmp/smolvm-kind-var-lib + containerPath: /var/lib/smolvm + - hostPath: /tmp/smolvm-kind-var-run + containerPath: /var/run/smolvm diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e9b46e..f314ad6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,10 +17,16 @@ jobs: go-version-file: go.mod cache: true - name: Run controller-runtime envtest - run: make test + run: make test-report + - name: Upload test reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: envtest-reports + path: reports/ kind-e2e: - name: kind e2e + name: kind topology e2e runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -31,56 +37,123 @@ jobs: - uses: helm/kind-action@v1 with: cluster_name: kind - - name: Run one deployment e2e test + - name: Run deployed-topology e2e test env: RUN_E2E: "true" KIND_CLUSTER: kind - run: make test-e2e + run: make test-e2e-report + - name: Upload e2e reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-reports + path: reports/ - smolvm-runtime-smoke: - name: smolvm runtime smoke + smolvm-runtime-e2e: + name: smolvm runtime lifecycle e2e runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 45 steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version-file: go.mod cache: true + - name: Prepare KVM-backed kind mounts + run: | + set -euxo pipefail + test -e /dev/kvm + sudo chmod 666 /dev/kvm + sudo mkdir -p /tmp/smolvm-kind-var-lib /tmp/smolvm-kind-var-run + sudo chmod 777 /tmp/smolvm-kind-var-lib /tmp/smolvm-kind-var-run - uses: helm/kind-action@v1 with: cluster_name: runtime - - name: Install smolvm + config: .github/kind-runtime-config.yaml + - name: Build and load operator images run: | - sudo apt-get update - sudo apt-get install -y libvirglrenderer1 libepoxy0 libgbm1 - curl -fsSL -o /tmp/smolvm.tar.gz https://github.com/smol-machines/smolvm/releases/download/v0.8.1/smolvm-0.8.1-linux-x86_64.tar.gz - mkdir -p /tmp/smolvm - tar -xzf /tmp/smolvm.tar.gz -C /tmp/smolvm --strip-components=1 - echo /tmp/smolvm >> "$GITHUB_PATH" - /tmp/smolvm/smolvm --version - - name: Run operator against real smolvm runtime + set -euxo pipefail + make docker-build IMG=example.com/smolvm-operator:e2e + make docker-build-runtime RUNTIME_IMG=example.com/smolvm-runtime:e2e + docker run --rm --entrypoint smolvm example.com/smolvm-runtime:e2e --version + docker run --rm --entrypoint /runtime-agent example.com/smolvm-runtime:e2e --help + kind load docker-image example.com/smolvm-operator:e2e --name runtime + kind load docker-image example.com/smolvm-runtime:e2e --name runtime + - name: Deploy full stack and run VM lifecycle run: | set -euxo pipefail - sudo chmod 666 /dev/kvm - export LD_LIBRARY_PATH="/tmp/smolvm/lib:${LD_LIBRARY_PATH:-}" - export LD_PRELOAD="$(find /usr/lib -name 'libvirglrenderer.so.1' | head -1)" - smolvm serve start --listen 127.0.0.1:8080 > smolvm-serve.log 2>&1 & - serve_pid=$! - trap 'kubectl delete smolvm runtime-smoke --ignore-not-found=true --wait=false || true; kill $serve_pid || true; cat smolvm-serve.log || true' EXIT - - for i in {1..60}; do - curl -fsS http://127.0.0.1:8080/api/v1/machines && break - sleep 1 - done + mkdir -p reports + current_step="install" + write_runtime_report() { + local status="$1" + local failed_step="${2:-}" + if [ "$status" = "pass" ]; then + cat > reports/runtime.xml <<'EOF' + + + + + + + + + + + + EOF + return + fi + cat > reports/runtime.xml < + + + + + EOF + } + cleanup() { + status=$? + if [ ! -f reports/runtime.xml ]; then + if [ "$status" -eq 0 ]; then + write_runtime_report pass + else + write_runtime_report fail "$current_step" + fi + fi + kubectl delete smolvm runtime-smoke --ignore-not-found=true --wait=false || true + kubectl -n operator-system get pods -o wide || true + kubectl -n operator-system logs deployment/operator-controller-manager --all-containers=true || true + kubectl -n operator-system logs daemonset/operator-smolvm-runtime --all-containers=true || true + exit "$status" + } + trap cleanup EXIT + current_step="install CRDs" make install - SMOLVM_API_URL=http://127.0.0.1:8080 go run ./cmd/main.go \ - --metrics-bind-address=0 \ - --health-probe-bind-address=:8081 > manager.log 2>&1 & - manager_pid=$! - trap 'kubectl delete smolvm runtime-smoke --ignore-not-found=true --wait=false || true; kill $manager_pid || true; kill $serve_pid || true; cat manager.log || true; cat smolvm-serve.log || true' EXIT + current_step="deploy full operator stack" + make deploy IMG=example.com/smolvm-operator:e2e RUNTIME_IMG=example.com/smolvm-runtime:e2e + + current_step="controller Deployment rollout" + kubectl rollout status deployment/operator-controller-manager -n operator-system --timeout=3m + current_step="runtime DaemonSet rollout" + kubectl rollout status daemonset/operator-smolvm-runtime -n operator-system --timeout=5m + current_step="SmolVMNode readiness" + node=$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}') + for i in {1..90}; do + kubectl get smolvmnode "$node" -o yaml || true + ready=$(kubectl get smolvmnode "$node" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true) + runtime_ready=$(kubectl get smolvmnode "$node" -o jsonpath='{.status.conditions[?(@.type=="RuntimeReady")].status}' 2>/dev/null || true) + if [ "$ready" = "True" ] && [ "$runtime_ready" = "True" ]; then + break + fi + sleep 2 + done + test "$(kubectl get smolvmnode "$node" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}')" = "True" + test "$(kubectl get smolvmnode "$node" -o jsonpath='{.status.conditions[?(@.type=="RuntimeReady")].status}')" = "True" + test "$(kubectl get smolvmnode "$node" -o jsonpath='{.status.endpoint.podNamespace}')" = "operator-system" + + current_step="create unpinned SmolVM" cat <<'EOF' | kubectl apply -f - apiVersion: vm.smolvm.dev/v1alpha1 kind: SmolVM @@ -94,40 +167,40 @@ jobs: memoryMiB: 128 EOF - ready=false - for i in {1..48}; do - echo "--- poll $i ---" + current_step="SmolVM reaches Running" + for i in {1..60}; do kubectl get smolvm runtime-smoke -o yaml || true - curl -fsS http://127.0.0.1:8080/api/v1/machines || true status=$(kubectl get smolvm runtime-smoke -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true) if [ "$status" = "True" ]; then - ready=true break fi sleep 10 done - if [ "$ready" != "true" ]; then - echo "runtime-smoke did not become Ready" - kubectl get smolvm runtime-smoke -o yaml || true - curl -fsS http://127.0.0.1:8080/api/v1/machines || true - exit 1 - fi + test "$(kubectl get smolvm runtime-smoke -o jsonpath='{.status.conditions[?(@.type=="Scheduled")].reason}')" = "Bound" test "$(kubectl get smolvm runtime-smoke -o jsonpath='{.status.conditions[?(@.type=="RuntimeReady")].status}')" = "True" - test "$(kubectl get smolvm runtime-smoke -o jsonpath='{.status.conditions[?(@.type=="GuestReady")].status}')" = "Unknown" + test "$(kubectl get smolvm runtime-smoke -o jsonpath='{.status.phase}')" = "Running" + current_step="patch spec.running false" kubectl patch smolvm runtime-smoke --type=merge -p '{"spec":{"running":false}}' - kubectl wait smolvm runtime-smoke \ - --for=jsonpath='{.status.phase}'=Stopped \ - --timeout=5m + current_step="SmolVM reaches Stopped" + kubectl wait smolvm runtime-smoke --for=jsonpath='{.status.phase}'=Stopped --timeout=5m + current_step="delete CR cleans runtime machine" machine=$(kubectl get smolvm runtime-smoke -o jsonpath='{.status.machineName}') + runtime_pod=$(kubectl -n operator-system get pod -l app.kubernetes.io/name=smolvm-runtime -o jsonpath='{.items[0].metadata.name}') kubectl delete smolvm runtime-smoke --wait=true --timeout=5m for i in {1..30}; do - if ! curl -fsS http://127.0.0.1:8080/api/v1/machines | grep -q "$machine"; then + if ! kubectl -n operator-system exec "$runtime_pod" -- smolvm machine ls | grep -q "$machine"; then exit 0 fi sleep 2 done echo "runtime machine $machine still exists after CR deletion" - curl -fsS http://127.0.0.1:8080/api/v1/machines || true + kubectl -n operator-system exec "$runtime_pod" -- smolvm machine ls || true exit 1 + - name: Upload runtime logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: runtime-e2e-logs + path: reports/ diff --git a/.gitignore b/.gitignore index 7a7feec..e47dbab 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ *.dylib bin/* Dockerfile.cross +reports/ # Test binary, built with `go test -c` *.test diff --git a/Dockerfile b/Dockerfile index 7c8ad90..7baf176 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ COPY go.sum go.sum RUN go mod download # Copy the go source -COPY cmd/main.go cmd/main.go +COPY cmd/ cmd/ COPY api/ api/ COPY internal/controller/ internal/controller/ COPY internal/smolvm/ internal/smolvm/ @@ -22,13 +22,15 @@ COPY internal/smolvm/ internal/smolvm/ # was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. -RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/main.go +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/main.go && \ + CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o runtime-agent cmd/runtime-agent/main.go # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details FROM gcr.io/distroless/static:nonroot WORKDIR / COPY --from=builder /workspace/manager . +COPY --from=builder /workspace/runtime-agent . USER 65532:65532 ENTRYPOINT ["/manager"] diff --git a/Dockerfile.runtime b/Dockerfile.runtime new file mode 100644 index 0000000..764f960 --- /dev/null +++ b/Dockerfile.runtime @@ -0,0 +1,35 @@ +FROM golang:1.25-bookworm AS agent-builder +WORKDIR /workspace +COPY go.mod go.mod +COPY go.sum go.sum +RUN go mod download +COPY cmd/ cmd/ +COPY api/ api/ +COPY internal/smolvm/ internal/smolvm/ +RUN CGO_ENABLED=0 go build -a -o /runtime-agent cmd/runtime-agent/main.go + +FROM debian:bookworm-slim AS smolvm-release +ARG TARGETARCH +ARG SMOLVM_VERSION=0.8.1 +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl tar && rm -rf /var/lib/apt/lists/* +WORKDIR /download +RUN case "${TARGETARCH}" in \ + amd64) SMOLVM_ARCH=x86_64 ;; \ + arm64) SMOLVM_ARCH=aarch64 ;; \ + *) echo "unsupported TARGETARCH=${TARGETARCH}" >&2; exit 1 ;; \ + esac && \ + curl -fsSL -o smolvm.tar.gz "https://github.com/smol-machines/smolvm/releases/download/v${SMOLVM_VERSION}/smolvm-${SMOLVM_VERSION}-linux-${SMOLVM_ARCH}.tar.gz" && \ + mkdir -p /opt/smolvm && \ + tar -xzf smolvm.tar.gz -C /opt/smolvm --strip-components=1 + +FROM ubuntu:24.04 +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends bash ca-certificates libepoxy0 libgbm1 libgcc-s1 libvirglrenderer1 && rm -rf /var/lib/apt/lists/* + +WORKDIR / +COPY --from=agent-builder /runtime-agent /runtime-agent +COPY --from=smolvm-release /opt/smolvm/ /opt/smolvm/ +RUN chmod 0755 /runtime-agent /opt/smolvm/smolvm /opt/smolvm/smolvm-bin && \ + ln -sf /opt/smolvm/smolvm /usr/local/bin/smolvm && \ + mkdir -p /var/lib/smolvm /var/run/smolvm + +ENTRYPOINT ["/runtime-agent"] diff --git a/Makefile b/Makefile index 451be71..07bbfa6 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,8 @@ # Image URL to use all building/pushing image targets IMG ?= controller:latest +RUNTIME_IMG ?= smolvm-runtime:latest +SMOLVM_VERSION ?= 0.8.1 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. ENVTEST_K8S_VERSION = 1.29.0 @@ -64,11 +66,21 @@ vet: ## Run go vet against code. test: manifests generate fmt vet envtest ## Run tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out +.PHONY: test-report +test-report: manifests generate fmt vet envtest gotestsum ## Run tests and write JUnit/coverage reports under reports/. + mkdir -p reports + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" $(GOTESTSUM) --format testname --junitfile reports/unit.xml -- -coverprofile reports/cover.out $$(go list ./... | grep -v /e2e) + # Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors. .PHONY: test-e2e # Run the e2e tests against a Kind k8s instance that is spun up. test-e2e: go test ./test/e2e/ -v -ginkgo.v +.PHONY: test-e2e-report +test-e2e-report: gotestsum ## Run e2e tests and write a JUnit report under reports/. + mkdir -p reports + $(GOTESTSUM) --format testname --junitfile reports/e2e.xml -- ./test/e2e/ -v -ginkgo.v + .PHONY: lint lint: golangci-lint ## Run golangci-lint linter & yamllint $(GOLANGCI_LINT) run @@ -82,6 +94,7 @@ lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes .PHONY: build build: manifests generate fmt vet ## Build manager binary. go build -o bin/manager cmd/main.go + go build -o bin/runtime-agent cmd/runtime-agent/main.go .PHONY: run run: manifests generate fmt vet ## Run a controller from your host. @@ -91,12 +104,17 @@ run: manifests generate fmt vet ## Run a controller from your host. # (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it. # More info: https://docs.docker.com/develop/develop-images/build_enhancements/ .PHONY: docker-build -docker-build: ## Build docker image with the manager. +docker-build: ## Build docker image with the manager and runtime-agent. $(CONTAINER_TOOL) build -t ${IMG} . +.PHONY: docker-build-runtime +docker-build-runtime: ## Build the smolvm runtime image, including the released smolvm binary. + $(CONTAINER_TOOL) build -f Dockerfile.runtime --build-arg SMOLVM_VERSION=$(SMOLVM_VERSION) -t ${RUNTIME_IMG} . + .PHONY: docker-push -docker-push: ## Push docker image with the manager. +docker-push: ## Push docker images. $(CONTAINER_TOOL) push ${IMG} + $(CONTAINER_TOOL) push ${RUNTIME_IMG} # PLATFORMS defines the target platforms for the manager image be built to provide support to multiple # architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to: @@ -123,6 +141,7 @@ build-installer: manifests generate kustomize ## Generate a consolidated YAML wi fi echo "---" >> dist/install.yaml # Add a document separator before appending cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + cd config/runtime && $(KUSTOMIZE) edit set image smolvm-runtime=${RUNTIME_IMG} $(KUSTOMIZE) build config/default >> dist/install.yaml ##@ Deployment @@ -142,6 +161,7 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified .PHONY: deploy deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + cd config/runtime && $(KUSTOMIZE) edit set image smolvm-runtime=${RUNTIME_IMG} $(KUSTOMIZE) build config/default | $(KUBECTL) apply -f - .PHONY: undeploy @@ -161,12 +181,14 @@ KUSTOMIZE ?= $(LOCALBIN)/kustomize-$(KUSTOMIZE_VERSION) CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen-$(CONTROLLER_TOOLS_VERSION) ENVTEST ?= $(LOCALBIN)/setup-envtest-$(ENVTEST_VERSION) GOLANGCI_LINT = $(LOCALBIN)/golangci-lint-$(GOLANGCI_LINT_VERSION) +GOTESTSUM ?= $(LOCALBIN)/gotestsum-$(GOTESTSUM_VERSION) ## Tool Versions KUSTOMIZE_VERSION ?= v5.3.0 CONTROLLER_TOOLS_VERSION ?= v0.19.0 ENVTEST_VERSION ?= latest GOLANGCI_LINT_VERSION ?= v1.54.2 +GOTESTSUM_VERSION ?= v1.13.0 .PHONY: kustomize kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. @@ -188,6 +210,11 @@ golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary. $(GOLANGCI_LINT): $(LOCALBIN) $(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint,${GOLANGCI_LINT_VERSION}) +.PHONY: gotestsum +gotestsum: $(GOTESTSUM) ## Download gotestsum locally if necessary. +$(GOTESTSUM): $(LOCALBIN) + $(call go-install-tool,$(GOTESTSUM),gotest.tools/gotestsum,${GOTESTSUM_VERSION}) + # go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist # $1 - target path with name of binary (ideally with version) # $2 - package url which can be installed diff --git a/README.md b/README.md index f36f364..de58efd 100644 --- a/README.md +++ b/README.md @@ -1,47 +1,80 @@ # smolvm operator -Kubernetes controller for declaratively managing node-local smolvm machines. +Kubernetes operator for declaratively managing node-local smolvm machines. + +The default install deploys a cluster-level `smolvm-controller` Deployment and a +node-local `smolvm-runtime` DaemonSet. Users do not run `smolvm serve` manually: +the runtime DaemonSet owns KVM access, local state, the smolvm socket, runtime +health, and `SmolVMNode` capability reporting. + +## Architecture + +```text +smolvm-controller Deployment + - schedules and reconciles SmolVMs + - writes SmolVM status and finalizers + - resolves node runtime endpoints from SmolVMNode + - calls the selected runtime over an authenticated node API + +smolvm-runtime DaemonSet + - runs on eligible nodes + - supervises smolvm serve + - owns /dev/kvm, /var/lib/smolvm, and /var/run/smolvm + - exposes the per-node runtime API + - reports SmolVMNode status +``` + +`SmolVMNode` is cluster-scoped. Object names match Kubernetes node names, and +`status.nodeUID` lets the controller detect stale endpoints after node +replacement. + +## Runtime API + +The controller calls the runtime endpoint advertised in `SmolVMNode.status`. +The default manifests use bearer-token authentication through the +`smolvm-runtime-auth` Secret. Production installs should replace the default +Secret value and may add certificate management and NetworkPolicy. -The operator reconciles `SmolVM` custom resources into calls to a node-local -`smolvm serve` API. It intentionally keeps Kubernetes reconciliation separate -from the privileged VM runtime: the controller owns desired state, while the -smolvm daemon owns KVM, libkrun, local disks, sockets, and machine processes. +Runtime operations are node-local and idempotent: -## Runtime contract +- `GetMachine` +- `CreateMachine` +- `EnsureMachineRunning` +- `StopMachine` +- `DeleteMachine` +- `ResizeMachine` +- `Health` +- `Identity` -Each node that runs `SmolVM` resources needs: +## Scheduling -- a running `smolvm serve` API -- KVM access (`/dev/kvm` on Linux) -- libkrun/libkrunfw and smolvm runtime assets -- a persistent host-local smolvm state directory -- protected API access, preferably through a Unix socket +`spec.nodeName` is optional. If omitted, the controller selects an eligible +`SmolVMNode` and records the immutable assignment in `status.nodeName`. -The controller reads these environment variables: +Rules: -| Variable | Default | Description | -| --- | --- | --- | -| `SMOLVM_API_URL` | `http://127.0.0.1:8080` | smolvm API base URL | -| `SMOLVM_API_SOCKET` | empty | Unix socket path; overrides network dialing when set | -| `SMOLVM_NODE_NAME` | empty | node identity for node-local reconciliation | +- `spec.nodeName` is an optional hard pin. +- `status.nodeName` is the actual binding. +- bound VMs do not move automatically. +- node/runtime loss updates status; it does not trigger silent migration. -## Current MVP behavior +## Deletion -The `SmolVM` reconciler supports: +Finalizers are node-aware. On delete, the controller calls the runtime endpoint +for `status.nodeName` and removes the finalizer only after local deletion +succeeds or the runtime reports the VM missing. -- stable smolvm machine names per CR -- finalizer-based delete cleanup -- create/start/stop/delete lifecycle -- image or `.smolmachine` source selection -- CPU/memory creation settings -- storage/overlay creation settings -- expand-only storage invariant checks -- outbound networking and host port mappings -- status phase and `Ready`/`Reconciled` conditions -- runtime-unavailable backoff +If the owning runtime is unavailable, deletion is blocked with a condition. The +escape hatch below removes the finalizer while accepting possible orphaned local +state: -This is an MVP controller, not a full pod runtime. Networking uses smolvm's -current API model; it does not provide Kubernetes CNI/Pod-IP semantics. +```yaml +metadata: + annotations: + vm.smolvm.dev/force-delete-local-state: "true" +``` + +When the runtime is available, the controller always attempts normal cleanup. ## Example @@ -53,6 +86,8 @@ metadata: spec: running: true image: alpine:latest + nodeSelector: + kubernetes.io/arch: amd64 resources: cpus: 1 memoryMiB: 512 @@ -78,17 +113,10 @@ Install CRDs: make install ``` -Run locally against a smolvm API: - -```sh -SMOLVM_API_URL=http://127.0.0.1:8080 make run -``` - -Or over a Unix socket: +Build and deploy the full operator stack: ```sh -SMOLVM_API_SOCKET=/var/run/smolvm/smolvm.sock make run +make docker-build IMG=/smolvm-operator: +make docker-build-runtime RUNTIME_IMG=/smolvm-runtime: +make deploy IMG=/smolvm-operator: RUNTIME_IMG=/smolvm-runtime: ``` - -The default Kubernetes DaemonSet manifest mounts `/var/run/smolvm` from each -node and talks to `/var/run/smolvm/smolvm.sock`. diff --git a/api/v1alpha1/smolvm_types.go b/api/v1alpha1/smolvm_types.go index 5a90e24..6e841fc 100644 --- a/api/v1alpha1/smolvm_types.go +++ b/api/v1alpha1/smolvm_types.go @@ -42,6 +42,10 @@ type SmolVMSpec struct { //+optional NodeName string `json:"nodeName,omitempty"` + // NodeSelector constrains automatic scheduling to nodes with matching labels. + //+optional + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + // Image is an OCI image to run inside the smolvm machine. //+kubebuilder:validation:MinLength=1 //+optional diff --git a/api/v1alpha1/smolvmnode_types.go b/api/v1alpha1/smolvmnode_types.go new file mode 100644 index 0000000..63344a6 --- /dev/null +++ b/api/v1alpha1/smolvmnode_types.go @@ -0,0 +1,125 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + +const ( + ConditionScheduled = "Scheduled" + ConditionDeletionBlocked = "DeletionBlocked" + + SmolVMNodeConditionReady = "Ready" + SmolVMNodeConditionKVMAvailable = "KVMAvailable" + SmolVMNodeConditionRuntimeReady = "RuntimeReady" + SmolVMNodeConditionSchedulable = "Schedulable" + + ForceDeleteLocalStateAnnotation = "vm.smolvm.dev/force-delete-local-state" +) + +// SmolVMNodeStatus defines node-local runtime capability and endpoint status. +type SmolVMNodeStatus struct { + // NodeUID is the Kubernetes Node UID observed by the node runtime. + //+optional + NodeUID string `json:"nodeUID,omitempty"` + + // RuntimeVersion is the smolvm runtime agent version. + //+optional + RuntimeVersion string `json:"runtimeVersion,omitempty"` + + // ProtocolVersion is the runtime API protocol version. + //+optional + ProtocolVersion string `json:"protocolVersion,omitempty"` + + // HeartbeatTime is the last time the runtime reported node status. + //+optional + HeartbeatTime *metav1.Time `json:"heartbeatTime,omitempty"` + + // Endpoint is the runtime API endpoint for this node. + //+optional + Endpoint SmolVMNodeEndpoint `json:"endpoint,omitempty"` + + // Allocatable reports VM resources available to smolvm on this node. + //+optional + Allocatable SmolVMNodeAllocatable `json:"allocatable,omitempty"` + + // Conditions advertise runtime readiness and scheduling capability. + //+optional + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// SmolVMNodeEndpoint identifies the node runtime API endpoint. +type SmolVMNodeEndpoint struct { + // PodName is the runtime DaemonSet pod name. + //+optional + PodName string `json:"podName,omitempty"` + + // PodNamespace is the runtime DaemonSet pod namespace. + //+optional + PodNamespace string `json:"podNamespace,omitempty"` + + // PodIP is the runtime pod IP address. + //+optional + PodIP string `json:"podIP,omitempty"` + + // Port is the authenticated runtime API port. + //+optional + Port int32 `json:"port,omitempty"` +} + +// SmolVMNodeAllocatable reports VM resource capacity. +type SmolVMNodeAllocatable struct { + // CPUs is the available vCPU count. + //+optional + CPUs int32 `json:"cpus,omitempty"` + + // MemoryMiB is the available memory in MiB. + //+optional + MemoryMiB int64 `json:"memoryMiB,omitempty"` + + // StorageGiB is the available local storage in GiB. + //+optional + StorageGiB int64 `json:"storageGiB,omitempty"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+kubebuilder:resource:scope=Cluster +//+kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status` +//+kubebuilder:printcolumn:name="Runtime",type=string,JSONPath=`.status.runtimeVersion` +//+kubebuilder:printcolumn:name="Heartbeat",type=date,JSONPath=`.status.heartbeatTime` +//+kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` + +// SmolVMNode is the Schema for node-local smolvm runtime capability. +type SmolVMNode struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Status SmolVMNodeStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// SmolVMNodeList contains a list of SmolVMNode. +type SmolVMNodeList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []SmolVMNode `json:"items"` +} + +func init() { + SchemeBuilder.Register(&SmolVMNode{}, &SmolVMNodeList{}) +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 167a21d..7549c2d 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -109,6 +109,122 @@ func (in *SmolVMNetwork) DeepCopy() *SmolVMNetwork { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SmolVMNode) DeepCopyInto(out *SmolVMNode) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SmolVMNode. +func (in *SmolVMNode) DeepCopy() *SmolVMNode { + if in == nil { + return nil + } + out := new(SmolVMNode) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SmolVMNode) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SmolVMNodeAllocatable) DeepCopyInto(out *SmolVMNodeAllocatable) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SmolVMNodeAllocatable. +func (in *SmolVMNodeAllocatable) DeepCopy() *SmolVMNodeAllocatable { + if in == nil { + return nil + } + out := new(SmolVMNodeAllocatable) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SmolVMNodeEndpoint) DeepCopyInto(out *SmolVMNodeEndpoint) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SmolVMNodeEndpoint. +func (in *SmolVMNodeEndpoint) DeepCopy() *SmolVMNodeEndpoint { + if in == nil { + return nil + } + out := new(SmolVMNodeEndpoint) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SmolVMNodeList) DeepCopyInto(out *SmolVMNodeList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]SmolVMNode, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SmolVMNodeList. +func (in *SmolVMNodeList) DeepCopy() *SmolVMNodeList { + if in == nil { + return nil + } + out := new(SmolVMNodeList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SmolVMNodeList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SmolVMNodeStatus) DeepCopyInto(out *SmolVMNodeStatus) { + *out = *in + if in.HeartbeatTime != nil { + in, out := &in.HeartbeatTime, &out.HeartbeatTime + *out = (*in).DeepCopy() + } + out.Endpoint = in.Endpoint + out.Allocatable = in.Allocatable + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SmolVMNodeStatus. +func (in *SmolVMNodeStatus) DeepCopy() *SmolVMNodeStatus { + if in == nil { + return nil + } + out := new(SmolVMNodeStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SmolVMPort) DeepCopyInto(out *SmolVMPort) { *out = *in @@ -142,6 +258,13 @@ func (in *SmolVMResources) DeepCopy() *SmolVMResources { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SmolVMSpec) DeepCopyInto(out *SmolVMSpec) { *out = *in + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } out.Resources = in.Resources out.Storage = in.Storage in.Network.DeepCopyInto(&out.Network) diff --git a/cmd/runtime-agent/main.go b/cmd/runtime-agent/main.go new file mode 100644 index 0000000..a08464d --- /dev/null +++ b/cmd/runtime-agent/main.go @@ -0,0 +1,500 @@ +package main + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "flag" + "fmt" + "math/big" + "net" + "net/http" + "os" + "os/exec" + "strconv" + "strings" + "syscall" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + vmv1alpha1 "github.com/manuschillerdev/smolvm-operator/api/v1alpha1" + smolvmapi "github.com/manuschillerdev/smolvm-operator/internal/smolvm" +) + +const protocolVersion = "v1alpha1" + +var scheme = runtime.NewScheme() + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(vmv1alpha1.AddToScheme(scheme)) +} + +//+kubebuilder:rbac:groups=vm.smolvm.dev,resources=smolvmnodes,verbs=get;list;watch;create;update;patch +//+kubebuilder:rbac:groups=vm.smolvm.dev,resources=smolvmnodes/status,verbs=get;update;patch +//+kubebuilder:rbac:groups="",resources=nodes,verbs=get + +func main() { + var bindAddr string + var socketPath string + var certFile string + var keyFile string + var smolvmServe string + var reportInterval time.Duration + var orphanCleanupPolicy string + flag.StringVar(&bindAddr, "bind-address", ":9443", "runtime API bind address") + flag.StringVar(&socketPath, "smolvm-api-socket", "/var/run/smolvm/smolvm.sock", "local smolvm serve Unix socket") + flag.StringVar(&certFile, "tls-cert-file", "", "TLS serving certificate") + flag.StringVar(&keyFile, "tls-private-key-file", "", "TLS private key") + flag.StringVar(&smolvmServe, "smolvm-serve-command", "smolvm serve start --listen unix:///var/run/smolvm/smolvm.sock", "command used to start smolvm serve; empty disables supervision") + flag.DurationVar(&reportInterval, "report-interval", 20*time.Second, "SmolVMNode status report interval") + flag.StringVar(&orphanCleanupPolicy, "orphan-cleanup-policy", "none", "local orphan cleanup policy: none or delete") + flag.Parse() + + ctrl.SetLogger(zap.New(zap.UseDevMode(true))) + ctx := ctrl.SetupSignalHandler() + + if strings.TrimSpace(smolvmServe) != "" { + go supervise(ctx, smolvmServe) + } + + cfg := ctrl.GetConfigOrDie() + k8s, err := client.New(cfg, client.Options{Scheme: scheme}) + if err != nil { + fatalErr(err) + } + + agent := &agent{ + client: k8s, + smolvm: smolvmapi.NewClient("http://unix", socketPath), + nodeName: getenv("NODE_NAME", ""), + podName: getenv("POD_NAME", ""), + podNS: getenv("POD_NAMESPACE", "default"), + podIP: getenv("POD_IP", ""), + version: getenv("SMOLVM_RUNTIME_VERSION", "unknown"), + token: getenv("SMOLVM_RUNTIME_TOKEN", ""), + listenPort: portFromAddr(bindAddr), + } + if agent.nodeName == "" { + fatalErr(fmt.Errorf("NODE_NAME is required")) + } + + go agent.reportLoop(ctx, reportInterval) + if orphanCleanupPolicy == "delete" { + go agent.orphanCleanupLoop(ctx, 5*time.Minute) + } + + mux := http.NewServeMux() + mux.HandleFunc("/healthz", agent.healthz) + mux.HandleFunc("/api/v1/identity", agent.withAuth(agent.identity)) + mux.HandleFunc("/api/v1/capabilities", agent.withAuth(agent.capabilities)) + mux.HandleFunc("/api/v1/machines", agent.withAuth(agent.machines)) + mux.HandleFunc("/api/v1/machines/", agent.withAuth(agent.machine)) + + server := &http.Server{Addr: bindAddr, Handler: mux, ReadHeaderTimeout: 5 * time.Second} + go func() { + <-ctx.Done() + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = server.Shutdown(shutdownCtx) + }() + if certFile != "" && keyFile != "" { + fatalErr(server.ListenAndServeTLS(certFile, keyFile)) + } + server.TLSConfig = selfSignedTLSConfig(agent.nodeName) + listener, err := tls.Listen("tcp", bindAddr, server.TLSConfig) + if err != nil { + fatalErr(err) + } + fatalErr(server.Serve(listener)) +} + +type agent struct { + client client.Client + smolvm *smolvmapi.Client + nodeName string + nodeUID string + podName string + podNS string + podIP string + version string + token string + listenPort int32 +} + +func (a *agent) withAuth(next http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if a.token != "" && r.Header.Get("Authorization") != "Bearer "+a.token { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + next(w, r) + } +} + +func (a *agent) healthz(w http.ResponseWriter, r *http.Request) { + health := a.health() + if !health.OK { + w.WriteHeader(http.StatusServiceUnavailable) + } + writeJSON(w, health) +} + +func (a *agent) health() smolvmapi.Health { + socketReady := pathExists("/var/run/smolvm/smolvm.sock") + stateReady := dirWritable("/var/lib/smolvm") + kvmAvailable := isCharDevice("/dev/kvm") + ok := socketReady && stateReady && kvmAvailable + message := "runtime healthy" + if !ok { + var missing []string + if !socketReady { + missing = append(missing, "smolvm socket unavailable") + } + if !stateReady { + missing = append(missing, "state directory unavailable") + } + if !kvmAvailable { + missing = append(missing, "KVM unavailable") + } + message = strings.Join(missing, "; ") + } + return smolvmapi.Health{OK: ok, KVMAvailable: kvmAvailable, SocketReady: socketReady, StateReady: stateReady, Message: message} +} + +func (a *agent) identity(w http.ResponseWriter, r *http.Request) { + writeJSON(w, smolvmapi.Identity{NodeName: a.nodeName, NodeUID: a.nodeUID}) +} + +func (a *agent) capabilities(w http.ResponseWriter, r *http.Request) { + writeJSON(w, a.capabilityReport()) +} + +func (a *agent) machines(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + machines, err := a.smolvm.ListMachines(r.Context()) + writeResult(w, smolvmapi.ListMachinesResponse{Machines: machines}, err) + return + } + if r.Method != http.MethodPost { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + var req smolvmapi.CreateMachineRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + machine, err := a.smolvm.CreateMachine(r.Context(), req) + writeResult(w, machine, err) +} + +func (a *agent) machine(w http.ResponseWriter, r *http.Request) { + parts := strings.Split(strings.TrimPrefix(r.URL.Path, "/api/v1/machines/"), "/") + if len(parts) == 0 || parts[0] == "" { + http.NotFound(w, r) + return + } + name := parts[0] + if len(parts) == 1 { + switch r.Method { + case http.MethodGet: + machine, err := a.smolvm.GetMachine(r.Context(), name) + writeResult(w, machine, err) + case http.MethodDelete: + writeResult(w, nil, a.smolvm.DeleteMachine(r.Context(), name)) + default: + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + } + return + } + if r.Method != http.MethodPost { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + var err error + switch parts[1] { + case "exec", "start": + err = a.smolvm.EnsureMachineRunning(r.Context(), name) + case "stop": + err = a.smolvm.StopMachine(r.Context(), name) + case "resize": + var req smolvmapi.ResizeRequest + if decErr := json.NewDecoder(r.Body).Decode(&req); decErr != nil { + http.Error(w, decErr.Error(), http.StatusBadRequest) + return + } + err = a.smolvm.ResizeMachine(r.Context(), name, req) + default: + http.NotFound(w, r) + return + } + writeResult(w, nil, err) +} + +func (a *agent) reportLoop(ctx context.Context, interval time.Duration) { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + a.report(ctx) + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} + +func (a *agent) report(ctx context.Context) { + var k8sNode corev1.Node + if err := a.client.Get(ctx, client.ObjectKey{Name: a.nodeName}, &k8sNode); err == nil { + a.nodeUID = string(k8sNode.UID) + } + now := metav1.Now() + node := &vmv1alpha1.SmolVMNode{ObjectMeta: metav1.ObjectMeta{Name: a.nodeName, Labels: k8sNode.Labels}} + _, _ = ctrl.CreateOrUpdate(ctx, a.client, node, func() error { return nil }) + latest := &vmv1alpha1.SmolVMNode{} + if err := a.client.Get(ctx, client.ObjectKey{Name: a.nodeName}, latest); err != nil { + return + } + latest.Labels = k8sNode.Labels + _ = a.client.Update(ctx, latest) + latest.Status.NodeUID = a.nodeUID + latest.Status.RuntimeVersion = a.version + latest.Status.ProtocolVersion = protocolVersion + latest.Status.HeartbeatTime = &now + latest.Status.Endpoint = vmv1alpha1.SmolVMNodeEndpoint{PodName: a.podName, PodNamespace: a.podNS, PodIP: a.podIP, Port: a.listenPort} + capabilities := a.capabilityReport() + latest.Status.Allocatable = vmv1alpha1.SmolVMNodeAllocatable{CPUs: capabilities.CPUs, MemoryMiB: capabilities.MemoryMiB, StorageGiB: capabilities.StorageGiB} + health := a.health() + setCondition(&latest.Status.Conditions, vmv1alpha1.SmolVMNodeConditionReady, conditionStatus(health.OK), "Reported", health.Message) + setCondition(&latest.Status.Conditions, vmv1alpha1.SmolVMNodeConditionKVMAvailable, conditionStatus(health.KVMAvailable), "Observed", "KVM device availability observed") + setCondition(&latest.Status.Conditions, vmv1alpha1.SmolVMNodeConditionRuntimeReady, conditionStatus(health.SocketReady), "Reported", health.Message) + setCondition(&latest.Status.Conditions, vmv1alpha1.SmolVMNodeConditionSchedulable, conditionStatus(kubernetesNodeReady(&k8sNode) && health.OK), "Reported", "node is schedulable for SmolVM") + _ = a.client.Status().Update(ctx, latest) +} + +func (a *agent) orphanCleanupLoop(ctx context.Context, interval time.Duration) { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + a.cleanupOrphans(ctx) + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} + +func (a *agent) cleanupOrphans(ctx context.Context) { + machines, err := a.smolvm.ListMachines(ctx) + if err != nil { + return + } + var vms vmv1alpha1.SmolVMList + if err := a.client.List(ctx, &vms); err != nil { + return + } + owned := map[string]struct{}{} + for _, vm := range vms.Items { + if vm.Status.NodeName != a.nodeName || vm.Status.MachineName == "" || !vm.DeletionTimestamp.IsZero() { + continue + } + owned[vm.Status.MachineName] = struct{}{} + } + for _, machine := range machines { + if !strings.HasPrefix(machine.Name, "k8s-") { + continue + } + if _, ok := owned[machine.Name]; ok { + continue + } + _ = a.smolvm.DeleteMachine(ctx, machine.Name) + } +} + +func supervise(ctx context.Context, command string) { + args := strings.Fields(command) + if len(args) == 0 { + return + } + for ctx.Err() == nil { + cmd := exec.CommandContext(ctx, args[0], args[1:]...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + _ = cmd.Run() + select { + case <-ctx.Done(): + return + case <-time.After(2 * time.Second): + } + } +} + +func setCondition(conditions *[]metav1.Condition, typ string, status metav1.ConditionStatus, reason, msg string) { + for i := range *conditions { + if (*conditions)[i].Type == typ { + if (*conditions)[i].Status != status { + (*conditions)[i].LastTransitionTime = metav1.Now() + } + (*conditions)[i].Status = status + (*conditions)[i].Reason = reason + (*conditions)[i].Message = msg + return + } + } + *conditions = append(*conditions, metav1.Condition{Type: typ, Status: status, Reason: reason, Message: msg, LastTransitionTime: metav1.Now()}) +} + +func selfSignedTLSConfig(nodeName string) *tls.Config { + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + fatalErr(err) + } + tmpl := x509.Certificate{ + SerialNumber: big.NewInt(time.Now().UnixNano()), + Subject: pkix.Name{CommonName: nodeName}, + NotBefore: time.Now().Add(-time.Minute), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + DNSNames: []string{nodeName}, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + } + certDER, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &key.PublicKey, key) + if err != nil { + fatalErr(err) + } + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}) + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}) + cert, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + fatalErr(err) + } + return &tls.Config{MinVersion: tls.VersionTLS12, Certificates: []tls.Certificate{cert}} +} + +func (a *agent) capabilityReport() smolvmapi.Capabilities { + cpus, memoryMiB := int32(0), int64(0) + var node corev1.Node + if err := a.client.Get(context.Background(), client.ObjectKey{Name: a.nodeName}, &node); err == nil { + cpus = int32(node.Status.Allocatable.Cpu().Value()) + memoryMiB = node.Status.Allocatable.Memory().Value() / 1024 / 1024 + } + return smolvmapi.Capabilities{ + RuntimeVersion: a.version, + ProtocolVersion: protocolVersion, + CPUs: cpus, + MemoryMiB: memoryMiB, + StorageGiB: storageGiB("/var/lib/smolvm"), + KVMAvailable: isCharDevice("/dev/kvm"), + } +} + +func kubernetesNodeReady(node *corev1.Node) bool { + for _, condition := range node.Status.Conditions { + if condition.Type == corev1.NodeReady { + return condition.Status == corev1.ConditionTrue + } + } + return false +} + +func writeResult(w http.ResponseWriter, out any, err error) { + if err != nil { + status := http.StatusInternalServerError + if smolvmapi.IsNotFound(err) || apierrors.IsNotFound(err) { + status = http.StatusNotFound + } + http.Error(w, err.Error(), status) + return + } + if out == nil { + w.WriteHeader(http.StatusNoContent) + return + } + writeJSON(w, out) +} + +func writeJSON(w http.ResponseWriter, out any) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(out) +} + +func conditionStatus(ok bool) metav1.ConditionStatus { + if ok { + return metav1.ConditionTrue + } + return metav1.ConditionFalse +} + +func isCharDevice(path string) bool { + info, err := os.Stat(path) + return err == nil && info.Mode()&os.ModeCharDevice != 0 +} + +func pathExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} + +func dirWritable(path string) bool { + info, err := os.Stat(path) + if err != nil || !info.IsDir() { + return false + } + file, err := os.CreateTemp(path, ".smolvm-write-check-*") + if err != nil { + return false + } + name := file.Name() + _ = file.Close() + _ = os.Remove(name) + return true +} + +func storageGiB(path string) int64 { + var stat syscall.Statfs_t + if err := syscall.Statfs(path, &stat); err != nil { + return 0 + } + return int64(stat.Bavail) * int64(stat.Bsize) / 1024 / 1024 / 1024 +} + +func getenv(key, fallback string) string { + if value := os.Getenv(key); value != "" { + return value + } + return fallback +} + +func portFromAddr(addr string) int32 { + idx := strings.LastIndex(addr, ":") + if idx < 0 { + return 0 + } + port, _ := strconv.Atoi(addr[idx+1:]) + return int32(port) +} + +func fatalErr(err error) { + if err != nil && err != http.ErrServerClosed { + fmt.Fprintln(os.Stderr, err.Error()) + os.Exit(1) + } +} diff --git a/config/crd/bases/vm.smolvm.dev_smolvmnodes.yaml b/config/crd/bases/vm.smolvm.dev_smolvmnodes.yaml new file mode 100644 index 0000000..bb64413 --- /dev/null +++ b/config/crd/bases/vm.smolvm.dev_smolvmnodes.yaml @@ -0,0 +1,168 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + name: smolvmnodes.vm.smolvm.dev +spec: + group: vm.smolvm.dev + names: + kind: SmolVMNode + listKind: SmolVMNodeList + plural: smolvmnodes + singular: smolvmnode + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .status.runtimeVersion + name: Runtime + type: string + - jsonPath: .status.heartbeatTime + name: Heartbeat + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: SmolVMNode is the Schema for node-local smolvm runtime capability. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + status: + description: SmolVMNodeStatus defines node-local runtime capability and + endpoint status. + properties: + allocatable: + description: Allocatable reports VM resources available to smolvm + on this node. + properties: + cpus: + description: CPUs is the available vCPU count. + format: int32 + type: integer + memoryMiB: + description: MemoryMiB is the available memory in MiB. + format: int64 + type: integer + storageGiB: + description: StorageGiB is the available local storage in GiB. + format: int64 + type: integer + type: object + conditions: + description: Conditions advertise runtime readiness and scheduling + capability. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + endpoint: + description: Endpoint is the runtime API endpoint for this node. + properties: + podIP: + description: PodIP is the runtime pod IP address. + type: string + podName: + description: PodName is the runtime DaemonSet pod name. + type: string + podNamespace: + description: PodNamespace is the runtime DaemonSet pod namespace. + type: string + port: + description: Port is the authenticated runtime API port. + format: int32 + type: integer + type: object + heartbeatTime: + description: HeartbeatTime is the last time the runtime reported node + status. + format: date-time + type: string + nodeUID: + description: NodeUID is the Kubernetes Node UID observed by the node + runtime. + type: string + protocolVersion: + description: ProtocolVersion is the runtime API protocol version. + type: string + runtimeVersion: + description: RuntimeVersion is the smolvm runtime agent version. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/vm.smolvm.dev_smolvms.yaml b/config/crd/bases/vm.smolvm.dev_smolvms.yaml index d4b96bb..2952d2a 100644 --- a/config/crd/bases/vm.smolvm.dev_smolvms.yaml +++ b/config/crd/bases/vm.smolvm.dev_smolvms.yaml @@ -108,6 +108,12 @@ spec: NodeName pins the machine to a Kubernetes node. Machines are node-local and must not move implicitly after creation. type: string + nodeSelector: + additionalProperties: + type: string + description: NodeSelector constrains automatic scheduling to nodes + with matching labels. + type: object resources: description: Resources configures VM CPU and memory. properties: diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 238be1e..53433d5 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -3,6 +3,7 @@ # It should be run by config/default resources: - bases/vm.smolvm.dev_smolvms.yaml +- bases/vm.smolvm.dev_smolvmnodes.yaml #+kubebuilder:scaffold:crdkustomizeresource patches: diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index f46b409..e2de526 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -18,6 +18,7 @@ resources: - ../crd - ../rbac - ../manager +- ../runtime # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml #- ../webhook diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml index 90649d4..501304b 100644 --- a/config/default/manager_auth_proxy_patch.yaml +++ b/config/default/manager_auth_proxy_patch.yaml @@ -1,7 +1,7 @@ # This patch inject a sidecar container which is a HTTP proxy for the # controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. apiVersion: apps/v1 -kind: DaemonSet +kind: Deployment metadata: name: controller-manager namespace: system diff --git a/config/default/manager_config_patch.yaml b/config/default/manager_config_patch.yaml index 1c9f0de..f6f5891 100644 --- a/config/default/manager_config_patch.yaml +++ b/config/default/manager_config_patch.yaml @@ -1,5 +1,5 @@ apiVersion: apps/v1 -kind: DaemonSet +kind: Deployment metadata: name: controller-manager namespace: system diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 14a461d..bc5beba 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -12,19 +12,20 @@ metadata: name: system --- apiVersion: apps/v1 -kind: DaemonSet +kind: Deployment metadata: name: controller-manager namespace: system labels: control-plane: controller-manager - app.kubernetes.io/name: daemonset + app.kubernetes.io/name: deployment app.kubernetes.io/instance: controller-manager app.kubernetes.io/component: manager app.kubernetes.io/created-by: operator app.kubernetes.io/part-of: operator app.kubernetes.io/managed-by: kustomize spec: + replicas: 1 selector: matchLabels: control-plane: controller-manager @@ -35,45 +36,22 @@ spec: labels: control-plane: controller-manager spec: - # TODO(user): Uncomment the following code to configure the nodeAffinity expression - # according to the platforms which are supported by your solution. - # It is considered best practice to support multiple architectures. You can - # build your manager image using the makefile target docker-buildx. - # affinity: - # nodeAffinity: - # requiredDuringSchedulingIgnoredDuringExecution: - # nodeSelectorTerms: - # - matchExpressions: - # - key: kubernetes.io/arch - # operator: In - # values: - # - amd64 - # - arm64 - # - ppc64le - # - s390x - # - key: kubernetes.io/os - # operator: In - # values: - # - linux securityContext: runAsNonRoot: true - # TODO(user): For common cases that do not require escalating privileges - # it is recommended to ensure that all your Pods/Containers are restrictive. - # More info: https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted - # Please uncomment the following code if your project does NOT have to work on old Kubernetes - # versions < 1.19 or on vendors versions which do NOT support this field by default (i.e. Openshift < 4.11 ). - # seccompProfile: - # type: RuntimeDefault + seccompProfile: + type: RuntimeDefault containers: - command: - /manager env: - - name: SMOLVM_NODE_NAME + - name: SMOLVM_RUNTIME_TOKEN valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: SMOLVM_API_SOCKET - value: /var/run/smolvm/smolvm.sock + secretKeyRef: + name: smolvm-runtime-auth + key: token + optional: true + - name: SMOLVM_RUNTIME_INSECURE_SKIP_VERIFY + value: "true" image: controller:latest name: manager securityContext: @@ -81,6 +59,9 @@ spec: capabilities: drop: - "ALL" + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 livenessProbe: httpGet: path: /healthz @@ -93,8 +74,6 @@ spec: port: 8081 initialDelaySeconds: 5 periodSeconds: 10 - # TODO(user): Configure the resources accordingly based on the project requirements. - # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ resources: limits: cpu: 500m @@ -102,14 +81,5 @@ spec: requests: cpu: 10m memory: 64Mi - volumeMounts: - - name: smolvm-run - mountPath: /var/run/smolvm - readOnly: false - volumes: - - name: smolvm-run - hostPath: - path: /var/run/smolvm - type: DirectoryOrCreate serviceAccountName: controller-manager terminationGracePeriodSeconds: 10 diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 14d5aac..1c8b400 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -11,13 +11,20 @@ rules: verbs: - create - patch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch - apiGroups: - vm.smolvm.dev resources: - - smolvms + - smolvmnodes verbs: - create - - delete - get - list - patch @@ -26,14 +33,27 @@ rules: - apiGroups: - vm.smolvm.dev resources: - - smolvms/finalizers + - smolvmnodes/status + - smolvms/status verbs: + - get + - patch - update - apiGroups: - vm.smolvm.dev resources: - - smolvms/status + - smolvms verbs: + - create + - delete - get + - list - patch - update + - watch +- apiGroups: + - vm.smolvm.dev + resources: + - smolvms/finalizers + verbs: + - update diff --git a/config/runtime/daemonset.yaml b/config/runtime/daemonset.yaml new file mode 100644 index 0000000..eae765a --- /dev/null +++ b/config/runtime/daemonset.yaml @@ -0,0 +1,97 @@ +apiVersion: v1 +kind: Secret +metadata: + name: smolvm-runtime-auth + namespace: system +type: Opaque +stringData: + token: change-me +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: smolvm-runtime + namespace: system + labels: + app.kubernetes.io/name: smolvm-runtime + app.kubernetes.io/component: runtime +spec: + selector: + matchLabels: + app.kubernetes.io/name: smolvm-runtime + template: + metadata: + labels: + app.kubernetes.io/name: smolvm-runtime + app.kubernetes.io/component: runtime + spec: + serviceAccountName: controller-manager + containers: + - name: runtime + image: smolvm-runtime:latest + command: + - /runtime-agent + args: + - --bind-address=:9443 + - --smolvm-api-socket=/var/run/smolvm/smolvm.sock + ports: + - name: runtime-api + containerPort: 9443 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: SMOLVM_RUNTIME_TOKEN + valueFrom: + secretKeyRef: + name: smolvm-runtime-auth + key: token + optional: true + securityContext: + privileged: true + readinessProbe: + httpGet: + path: /healthz + port: 9443 + scheme: HTTPS + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /healthz + port: 9443 + scheme: HTTPS + initialDelaySeconds: 15 + periodSeconds: 20 + volumeMounts: + - name: kvm + mountPath: /dev/kvm + - name: state + mountPath: /var/lib/smolvm + - name: run + mountPath: /var/run/smolvm + volumes: + - name: kvm + hostPath: + path: /dev/kvm + - name: state + hostPath: + path: /var/lib/smolvm + type: DirectoryOrCreate + - name: run + hostPath: + path: /var/run/smolvm + type: DirectoryOrCreate diff --git a/config/runtime/kustomization.yaml b/config/runtime/kustomization.yaml new file mode 100644 index 0000000..b4c7110 --- /dev/null +++ b/config/runtime/kustomization.yaml @@ -0,0 +1,8 @@ +resources: +- daemonset.yaml +- networkpolicy.yaml + +images: +- name: smolvm-runtime + newName: smolvm-runtime + newTag: latest diff --git a/config/runtime/networkpolicy.yaml b/config/runtime/networkpolicy.yaml new file mode 100644 index 0000000..2c4d1ef --- /dev/null +++ b/config/runtime/networkpolicy.yaml @@ -0,0 +1,20 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: smolvm-runtime-api + namespace: system +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: smolvm-runtime + app.kubernetes.io/component: runtime + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + control-plane: controller-manager + ports: + - protocol: TCP + port: 9443 diff --git a/config/samples/vm_v1alpha1_smolvm.yaml b/config/samples/vm_v1alpha1_smolvm.yaml index 0b74331..3559811 100644 --- a/config/samples/vm_v1alpha1_smolvm.yaml +++ b/config/samples/vm_v1alpha1_smolvm.yaml @@ -9,8 +9,6 @@ metadata: app.kubernetes.io/created-by: operator name: smolvm-sample spec: - # Required when the controller runs node-local with SMOLVM_NODE_NAME set. - nodeName: worker-1 running: true image: alpine:latest resources: diff --git a/internal/controller/smolvm_controller.go b/internal/controller/smolvm_controller.go index e2e1caf..5a5ede5 100644 --- a/internal/controller/smolvm_controller.go +++ b/internal/controller/smolvm_controller.go @@ -24,8 +24,12 @@ import ( "net" "os" "sort" + "strings" "time" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apiequality "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -34,7 +38,9 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" vmv1alpha1 "github.com/manuschillerdev/smolvm-operator/api/v1alpha1" smolvmapi "github.com/manuschillerdev/smolvm-operator/internal/smolvm" @@ -52,6 +58,10 @@ type SmolVMReconciler struct { } type SmolVMRuntime interface { + Health(context.Context) (*smolvmapi.Health, error) + GetIdentity(context.Context) (*smolvmapi.Identity, error) + Capabilities(context.Context) (*smolvmapi.Capabilities, error) + ListMachines(context.Context) ([]smolvmapi.MachineInfo, error) GetMachine(context.Context, string) (*smolvmapi.MachineInfo, error) CreateMachine(context.Context, smolvmapi.CreateMachineRequest) (*smolvmapi.MachineInfo, error) EnsureMachineRunning(context.Context, string) error @@ -67,9 +77,51 @@ func (r *SmolVMReconciler) runtimeClient() SmolVMRuntime { return smolvmapi.NewClient(os.Getenv("SMOLVM_API_URL"), os.Getenv("SMOLVM_API_SOCKET")) } +func (r *SmolVMReconciler) runtimeForNode(ctx context.Context, nodeName string) (SmolVMRuntime, error) { + if r.RuntimeFactory != nil { + return r.RuntimeFactory(), nil + } + var node vmv1alpha1.SmolVMNode + if err := r.Get(ctx, types.NamespacedName{Name: nodeName}, &node); err != nil { + return nil, fmt.Errorf("get SmolVMNode %s: %w", nodeName, err) + } + if !smolVMNodeEligible(&node) { + return nil, fmt.Errorf("SmolVMNode %s is not ready or heartbeat is stale", nodeName) + } + endpoint := node.Status.Endpoint + if endpoint.PodIP == "" || endpoint.Port == 0 { + return nil, fmt.Errorf("SmolVMNode %s has no runtime endpoint", nodeName) + } + api := smolvmapi.NewClientWithAuth(fmt.Sprintf("https://%s:%d", endpoint.PodIP, endpoint.Port), "", os.Getenv("SMOLVM_RUNTIME_TOKEN"), os.Getenv("SMOLVM_RUNTIME_INSECURE_SKIP_VERIFY") == "true") + health, err := api.Health(ctx) + if err != nil { + return nil, fmt.Errorf("get runtime health for node %s: %w", nodeName, err) + } + if !health.OK { + return nil, fmt.Errorf("runtime for node %s is unhealthy: %s", nodeName, health.Message) + } + identity, err := api.GetIdentity(ctx) + if err != nil { + return nil, fmt.Errorf("get runtime identity for node %s: %w", nodeName, err) + } + if identity.NodeName != nodeName || (node.Status.NodeUID != "" && identity.NodeUID != "" && identity.NodeUID != node.Status.NodeUID) { + return nil, fmt.Errorf("runtime endpoint identity mismatch for node %s: got node=%s uid=%s", nodeName, identity.NodeName, identity.NodeUID) + } + capabilities, err := api.Capabilities(ctx) + if err != nil { + return nil, fmt.Errorf("get runtime capabilities for node %s: %w", nodeName, err) + } + if capabilities.ProtocolVersion != "" && capabilities.ProtocolVersion != node.Status.ProtocolVersion { + return nil, fmt.Errorf("runtime protocol mismatch for node %s: endpoint=%s status=%s", nodeName, capabilities.ProtocolVersion, node.Status.ProtocolVersion) + } + return api, nil +} + //+kubebuilder:rbac:groups=vm.smolvm.dev,resources=smolvms,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=vm.smolvm.dev,resources=smolvms/status,verbs=get;update;patch //+kubebuilder:rbac:groups=vm.smolvm.dev,resources=smolvms/finalizers,verbs=update +//+kubebuilder:rbac:groups=vm.smolvm.dev,resources=smolvmnodes,verbs=get;list;watch +//+kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch //+kubebuilder:rbac:groups="",resources=events,verbs=create;patch func (r *SmolVMReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { @@ -80,40 +132,15 @@ func (r *SmolVMReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr return ctrl.Result{}, client.IgnoreNotFound(err) } - runtimeNode := os.Getenv("SMOLVM_NODE_NAME") - desiredNode := vm.Spec.NodeName ownerNode := vm.Status.NodeName - if ownerNode == "" { - ownerNode = desiredNode - } machineName := vm.Status.MachineName if machineName == "" { machineName = stableMachineName(&vm) } - if runtimeNode != "" && ownerNode == "" { - return r.updateStatus(ctx, &vm, statusInput{ - Phase: "Pending", - NodeName: "", - MachineName: machineName, - Ready: metav1.ConditionFalse, - ReadyReason: "AwaitingNodeAssignment", - ReadyMsg: "spec.nodeName is required in node-local mode", - Recon: metav1.ConditionFalse, - ReconReason: "AwaitingNodeAssignment", - ReconMsg: "set spec.nodeName to a node running the smolvm controller", - }) - } - - if ownerNode != "" && runtimeNode != "" && ownerNode != runtimeNode { - return ctrl.Result{}, nil - } - - api := r.runtimeClient() - if !vm.ObjectMeta.DeletionTimestamp.IsZero() { - return r.reconcileDelete(ctx, &vm, api, machineName, ownerNode) + return r.reconcileDelete(ctx, &vm, machineName, ownerNode) } if !controllerutil.ContainsFinalizer(&vm, vmv1alpha1.SmolVMFinalizer) { @@ -137,6 +164,52 @@ func (r *SmolVMReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr }) } + if ownerNode == "" { + nodeName, msg, err := r.schedule(ctx, &vm) + if err != nil { + return ctrl.Result{}, err + } + if nodeName == "" { + return r.updateStatus(ctx, &vm, statusInput{ + Phase: "Pending", + NodeName: "", + MachineName: machineName, + Scheduled: metav1.ConditionFalse, + ScheduledReason: "NoEligibleNodes", + ScheduledMsg: msg, + Ready: metav1.ConditionFalse, + ReadyReason: "AwaitingNodeAssignment", + ReadyMsg: msg, + Recon: metav1.ConditionFalse, + ReconReason: "AwaitingNodeAssignment", + ReconMsg: msg, + }) + } + _, statusErr := r.updateStatus(ctx, &vm, statusInput{ + Phase: "Scheduled", + NodeName: nodeName, + MachineName: machineName, + Scheduled: metav1.ConditionTrue, + ScheduledReason: "Bound", + ScheduledMsg: fmt.Sprintf("assigned to node %s", nodeName), + Ready: metav1.ConditionFalse, + ReadyReason: "Scheduled", + ReadyMsg: "waiting for node runtime reconciliation", + Recon: metav1.ConditionFalse, + ReconReason: "RuntimePending", + ReconMsg: "waiting for node runtime reconciliation", + }) + if statusErr != nil { + return ctrl.Result{}, statusErr + } + return ctrl.Result{RequeueAfter: requeueAfter}, nil + } + + api, err := r.runtimeForNode(ctx, ownerNode) + if err != nil { + return r.runtimeUnavailable(ctx, &vm, ownerNode, machineName, err) + } + machine, err := api.GetMachine(ctx, machineName) if smolvmapi.IsNotFound(err) { createReq := buildCreateRequest(&vm, machineName) @@ -257,12 +330,46 @@ func (r *SmolVMReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr return ctrl.Result{RequeueAfter: requeueAfter}, nil } -func (r *SmolVMReconciler) reconcileDelete(ctx context.Context, vm *vmv1alpha1.SmolVM, api SmolVMRuntime, machineName, ownerNode string) (ctrl.Result, error) { - if machineName != "" { - r.event(vm, "Normal", "Deleting", fmt.Sprintf("deleting smolvm machine %s", machineName)) - if err := api.DeleteMachine(ctx, machineName); err != nil && !smolvmapi.IsNotFound(err) { - return r.runtimeUnavailable(ctx, vm, ownerNode, machineName, err) +func (r *SmolVMReconciler) reconcileDelete(ctx context.Context, vm *vmv1alpha1.SmolVM, machineName, ownerNode string) (ctrl.Result, error) { + if ownerNode == "" || machineName == "" { + controllerutil.RemoveFinalizer(vm, vmv1alpha1.SmolVMFinalizer) + return ctrl.Result{}, r.Update(ctx, vm) + } + + api, err := r.runtimeForNode(ctx, ownerNode) + if err != nil { + if vm.Annotations[vmv1alpha1.ForceDeleteLocalStateAnnotation] == "true" { + r.event(vm, "Warning", "ForceDeleteLocalState", fmt.Sprintf("removing finalizer while runtime for node %s is unavailable; local VM state may remain", ownerNode)) + controllerutil.RemoveFinalizer(vm, vmv1alpha1.SmolVMFinalizer) + return ctrl.Result{}, r.Update(ctx, vm) + } + r.event(vm, "Warning", "DeletionBlocked", err.Error()) + _, statusErr := r.updateStatus(ctx, vm, statusInput{ + Phase: "Deleting", + NodeName: ownerNode, + MachineName: machineName, + Scheduled: metav1.ConditionTrue, + ScheduledReason: "Bound", + ScheduledMsg: fmt.Sprintf("assigned to node %s", ownerNode), + Ready: metav1.ConditionFalse, + ReadyReason: "RuntimeUnavailable", + ReadyMsg: err.Error(), + Recon: metav1.ConditionFalse, + ReconReason: "DeletionBlocked", + ReconMsg: err.Error(), + DeletionBlocked: metav1.ConditionTrue, + DeletionBlockedReason: "RuntimeUnavailable", + DeletionBlockedMsg: fmt.Sprintf("owning node %s is unavailable; local VM state may still exist", ownerNode), + }) + if statusErr != nil { + return ctrl.Result{}, statusErr } + return ctrl.Result{RequeueAfter: requeueAfter}, nil + } + + r.event(vm, "Normal", "Deleting", fmt.Sprintf("deleting smolvm machine %s", machineName)) + if err := api.DeleteMachine(ctx, machineName); err != nil && !smolvmapi.IsNotFound(err) { + return r.runtimeUnavailable(ctx, vm, ownerNode, machineName, err) } controllerutil.RemoveFinalizer(vm, vmv1alpha1.SmolVMFinalizer) return ctrl.Result{}, r.Update(ctx, vm) @@ -288,16 +395,22 @@ func (r *SmolVMReconciler) runtimeUnavailable(ctx context.Context, vm *vmv1alpha } type statusInput struct { - Phase string - NodeName string - MachineName string - Machine *smolvmapi.MachineInfo - Ready metav1.ConditionStatus - ReadyReason string - ReadyMsg string - Recon metav1.ConditionStatus - ReconReason string - ReconMsg string + Phase string + NodeName string + MachineName string + Machine *smolvmapi.MachineInfo + Scheduled metav1.ConditionStatus + ScheduledReason string + ScheduledMsg string + Ready metav1.ConditionStatus + ReadyReason string + ReadyMsg string + Recon metav1.ConditionStatus + ReconReason string + ReconMsg string + DeletionBlocked metav1.ConditionStatus + DeletionBlockedReason string + DeletionBlockedMsg string } func (r *SmolVMReconciler) updateStatus(ctx context.Context, vm *vmv1alpha1.SmolVM, in statusInput) (ctrl.Result, error) { @@ -317,10 +430,21 @@ func (r *SmolVMReconciler) updateStatus(ctx context.Context, vm *vmv1alpha1.Smol latest.Status.OverlayGiB = in.Machine.OverlayGB } latest.Status.ObservedGeneration = latest.Generation + if in.ScheduledReason == "" && in.NodeName != "" { + in.Scheduled = metav1.ConditionTrue + in.ScheduledReason = "Bound" + in.ScheduledMsg = fmt.Sprintf("assigned to node %s", in.NodeName) + } + if in.ScheduledReason != "" { + setCondition(&latest.Status.Conditions, vmv1alpha1.ConditionScheduled, in.Scheduled, in.ScheduledReason, in.ScheduledMsg, latest.Generation) + } setCondition(&latest.Status.Conditions, vmv1alpha1.ConditionReady, in.Ready, in.ReadyReason, in.ReadyMsg, latest.Generation) setCondition(&latest.Status.Conditions, vmv1alpha1.ConditionRuntimeReady, in.Ready, in.ReadyReason, in.ReadyMsg, latest.Generation) setCondition(&latest.Status.Conditions, vmv1alpha1.ConditionGuestReady, metav1.ConditionUnknown, "GuestReadinessUnavailable", "smolvm guest readiness is not reported by the runtime API", latest.Generation) setCondition(&latest.Status.Conditions, vmv1alpha1.ConditionReconciled, in.Recon, in.ReconReason, in.ReconMsg, latest.Generation) + if in.DeletionBlockedReason != "" { + setCondition(&latest.Status.Conditions, vmv1alpha1.ConditionDeletionBlocked, in.DeletionBlocked, in.DeletionBlockedReason, in.DeletionBlockedMsg, latest.Generation) + } if apiequality.Semantic.DeepEqual(previous, &latest.Status) { return ctrl.Result{}, nil } @@ -350,6 +474,171 @@ func setCondition(conditions *[]metav1.Condition, typ string, status metav1.Cond }) } +func (r *SmolVMReconciler) schedule(ctx context.Context, vm *vmv1alpha1.SmolVM) (string, string, error) { + bound, err := r.boundVMs(ctx, vm) + if err != nil { + return "", "", err + } + if vm.Spec.NodeName != "" { + var node vmv1alpha1.SmolVMNode + if err := r.Get(ctx, types.NamespacedName{Name: vm.Spec.NodeName}, &node); err != nil { + if apierrors.IsNotFound(err) { + return "", fmt.Sprintf("pinned node %s has no SmolVMNode runtime status", vm.Spec.NodeName), nil + } + return "", "", err + } + if reason := r.nodeRejectReason(ctx, &node, vm, bound); reason != "" { + return "", fmt.Sprintf("pinned node %s is not eligible: %s", vm.Spec.NodeName, reason), nil + } + return node.Name, "", nil + } + + var nodes vmv1alpha1.SmolVMNodeList + if err := r.List(ctx, &nodes); err != nil { + return "", "", err + } + var reasons []string + for _, node := range nodes.Items { + if reason := r.nodeRejectReason(ctx, &node, vm, bound); reason != "" { + reasons = append(reasons, fmt.Sprintf("%s: %s", node.Name, reason)) + continue + } + return node.Name, "", nil + } + if len(reasons) == 0 { + return "", "no SmolVMNode objects exist", nil + } + return "", "no ready SmolVMNode matches this VM: " + strings.Join(reasons, "; "), nil +} + +func (r *SmolVMReconciler) nodeRejectReason(ctx context.Context, node *vmv1alpha1.SmolVMNode, vm *vmv1alpha1.SmolVM, bound []vmv1alpha1.SmolVM) string { + if !smolVMNodeEligible(node) { + return "runtime not ready, heartbeat stale, or endpoint missing" + } + if !matchesNodeSelector(node, vm.Spec.NodeSelector) { + return "nodeSelector does not match" + } + var k8sNode corev1.Node + if err := r.Get(ctx, types.NamespacedName{Name: node.Name}, &k8sNode); err != nil { + return fmt.Sprintf("Kubernetes Node unavailable: %v", err) + } + if k8sNode.Spec.Unschedulable { + return "Kubernetes Node is unschedulable" + } + if !kubernetesNodeReady(&k8sNode) { + return "Kubernetes Node is not Ready" + } + if reason := resourceFitReason(node, vm, bound); reason != "" { + return reason + } + if reason := hostPortConflictReason(node.Name, vm, bound); reason != "" { + return reason + } + return "" +} + +func (r *SmolVMReconciler) boundVMs(ctx context.Context, pending *vmv1alpha1.SmolVM) ([]vmv1alpha1.SmolVM, error) { + var list vmv1alpha1.SmolVMList + if err := r.List(ctx, &list); err != nil { + return nil, err + } + out := make([]vmv1alpha1.SmolVM, 0, len(list.Items)) + for _, item := range list.Items { + if item.UID == pending.UID || item.Status.NodeName == "" || !item.DeletionTimestamp.IsZero() { + continue + } + out = append(out, item) + } + return out, nil +} + +func kubernetesNodeReady(node *corev1.Node) bool { + for _, condition := range node.Status.Conditions { + if condition.Type == corev1.NodeReady { + return condition.Status == corev1.ConditionTrue + } + } + return false +} + +func resourceFitReason(node *vmv1alpha1.SmolVMNode, vm *vmv1alpha1.SmolVM, bound []vmv1alpha1.SmolVM) string { + usedCPU, usedMemory, usedStorage := int32(0), int64(0), int64(0) + for _, existing := range bound { + if existing.Status.NodeName != node.Name { + continue + } + usedCPU += existing.Spec.Resources.CPUs + usedMemory += int64(existing.Spec.Resources.MemoryMiB) + usedStorage += existing.Spec.Storage.StorageGiB + existing.Spec.Storage.OverlayGiB + } + if node.Status.Allocatable.CPUs > 0 && usedCPU+vm.Spec.Resources.CPUs > node.Status.Allocatable.CPUs { + return fmt.Sprintf("insufficient cpu: requested %d used %d allocatable %d", vm.Spec.Resources.CPUs, usedCPU, node.Status.Allocatable.CPUs) + } + if node.Status.Allocatable.MemoryMiB > 0 && usedMemory+int64(vm.Spec.Resources.MemoryMiB) > node.Status.Allocatable.MemoryMiB { + return fmt.Sprintf("insufficient memory: requested %dMiB used %dMiB allocatable %dMiB", vm.Spec.Resources.MemoryMiB, usedMemory, node.Status.Allocatable.MemoryMiB) + } + requestedStorage := vm.Spec.Storage.StorageGiB + vm.Spec.Storage.OverlayGiB + if node.Status.Allocatable.StorageGiB > 0 && usedStorage+requestedStorage > node.Status.Allocatable.StorageGiB { + return fmt.Sprintf("insufficient storage: requested %dGiB used %dGiB allocatable %dGiB", requestedStorage, usedStorage, node.Status.Allocatable.StorageGiB) + } + return "" +} + +func hostPortConflictReason(nodeName string, vm *vmv1alpha1.SmolVM, bound []vmv1alpha1.SmolVM) string { + requested := map[int32]struct{}{} + for _, port := range vm.Spec.Network.Ports { + requested[port.HostPort] = struct{}{} + } + if len(requested) == 0 { + return "" + } + for _, existing := range bound { + if existing.Status.NodeName != nodeName { + continue + } + for _, port := range existing.Spec.Network.Ports { + if _, ok := requested[port.HostPort]; ok { + return fmt.Sprintf("hostPort %d conflicts with %s/%s", port.HostPort, existing.Namespace, existing.Name) + } + } + } + return "" +} + +func smolVMNodeEligible(node *vmv1alpha1.SmolVMNode) bool { + if !hasCondition(node.Status.Conditions, vmv1alpha1.SmolVMNodeConditionReady, metav1.ConditionTrue) || + !hasCondition(node.Status.Conditions, vmv1alpha1.SmolVMNodeConditionKVMAvailable, metav1.ConditionTrue) || + !hasCondition(node.Status.Conditions, vmv1alpha1.SmolVMNodeConditionRuntimeReady, metav1.ConditionTrue) || + !hasCondition(node.Status.Conditions, vmv1alpha1.SmolVMNodeConditionSchedulable, metav1.ConditionTrue) { + return false + } + if node.Status.ProtocolVersion != "" && node.Status.ProtocolVersion != "v1alpha1" { + return false + } + if node.Status.HeartbeatTime == nil || time.Since(node.Status.HeartbeatTime.Time) > 2*time.Minute { + return false + } + return node.Status.Endpoint.PodIP != "" && node.Status.Endpoint.Port > 0 +} + +func hasCondition(conditions []metav1.Condition, typ string, status metav1.ConditionStatus) bool { + for _, condition := range conditions { + if condition.Type == typ && condition.Status == status { + return true + } + } + return false +} + +func matchesNodeSelector(node *vmv1alpha1.SmolVMNode, selector map[string]string) bool { + for key, value := range selector { + if node.Labels[key] != value { + return false + } + } + return true +} + func validateSpec(vm *vmv1alpha1.SmolVM) error { if vm.Status.NodeName != "" && vm.Spec.NodeName != "" && vm.Spec.NodeName != vm.Status.NodeName { return fmt.Errorf("spec.nodeName is immutable after binding; machine is owned by node %q", vm.Status.NodeName) @@ -501,6 +790,20 @@ func (r *SmolVMReconciler) event(vm *vmv1alpha1.SmolVM, eventType, reason, messa } } +func (r *SmolVMReconciler) requestsForNodeChange(ctx context.Context, nodeName string) []reconcile.Request { + var vms vmv1alpha1.SmolVMList + if err := r.List(ctx, &vms); err != nil { + return nil + } + requests := make([]reconcile.Request, 0, len(vms.Items)) + for _, vm := range vms.Items { + if vm.Status.NodeName == nodeName || (vm.Status.NodeName == "" && (vm.Spec.NodeName == "" || vm.Spec.NodeName == nodeName)) { + requests = append(requests, reconcile.Request{NamespacedName: types.NamespacedName{Name: vm.Name, Namespace: vm.Namespace}}) + } + } + return requests +} + // SetupWithManager sets up the controller with the Manager. func (r *SmolVMReconciler) SetupWithManager(mgr ctrl.Manager) error { if r.Recorder == nil { @@ -508,5 +811,11 @@ func (r *SmolVMReconciler) SetupWithManager(mgr ctrl.Manager) error { } return ctrl.NewControllerManagedBy(mgr). For(&vmv1alpha1.SmolVM{}). + Watches(&corev1.Node{}, handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []reconcile.Request { + return r.requestsForNodeChange(ctx, obj.GetName()) + })). + Watches(&vmv1alpha1.SmolVMNode{}, handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []reconcile.Request { + return r.requestsForNodeChange(ctx, obj.GetName()) + })). Complete(r) } diff --git a/internal/controller/smolvm_controller_test.go b/internal/controller/smolvm_controller_test.go index aeff50c..eeb6490 100644 --- a/internal/controller/smolvm_controller_test.go +++ b/internal/controller/smolvm_controller_test.go @@ -2,10 +2,18 @@ package controller import ( "context" + "encoding/json" "fmt" + "net" "net/http" + "net/http/httptest" + "net/url" "os" + "strconv" "testing" + "time" + + corev1 "k8s.io/api/core/v1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -105,19 +113,112 @@ func TestBuildCreateRequestMapsSpec(t *testing.T) { } var _ = Describe("SmolVM Controller", func() { + It("schedules an unpinned VM to a ready SmolVMNode and calls that runtime endpoint", func() { + ctx := context.Background() + runtime := newRuntimeHTTPServer("node-schedule", "uid-schedule") + defer runtime.Close() + Expect(createReadyNode(ctx, "node-schedule", "uid-schedule")).To(Succeed()) + Expect(createReadySmolVMNode(ctx, "node-schedule", "uid-schedule", runtime.host(), runtime.port(), vmv1alpha1.SmolVMNodeAllocatable{CPUs: 4, MemoryMiB: 4096, StorageGiB: 100})).To(Succeed()) + + name := types.NamespacedName{Name: "schedule-unpinned", Namespace: "default"} + Expect(k8sClient.Create(ctx, testSmolVM(name, ""))).To(Succeed()) + reconciler := clusterReconciler() + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: name}) + Expect(err).NotTo(HaveOccurred()) + _, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: name}) + Expect(err).NotTo(HaveOccurred()) + _, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: name}) + Expect(err).NotTo(HaveOccurred()) + + latest := &vmv1alpha1.SmolVM{} + Expect(k8sClient.Get(ctx, name, latest)).To(Succeed()) + Expect(latest.Status.NodeName).To(Equal("node-schedule")) + Expect(conditionReason(latest, vmv1alpha1.ConditionScheduled)).To(Equal("Bound")) + Expect(runtime.created).To(Equal(1)) + }) + + It("does not schedule when resource fit or host ports would conflict", func() { + ctx := context.Background() + runtime := newRuntimeHTTPServer("node-fit", "uid-fit") + defer runtime.Close() + Expect(createReadyNode(ctx, "node-fit", "uid-fit")).To(Succeed()) + Expect(createReadySmolVMNode(ctx, "node-fit", "uid-fit", runtime.host(), runtime.port(), vmv1alpha1.SmolVMNodeAllocatable{CPUs: 4, MemoryMiB: 4096, StorageGiB: 10})).To(Succeed()) + + existingName := types.NamespacedName{Name: "bound-port", Namespace: "default"} + existing := testSmolVM(existingName, "") + existing.Spec.Network.Ports = []vmv1alpha1.SmolVMPort{{HostPort: 8080, GuestPort: 80}} + Expect(createSmolVMWithSpecStatus(ctx, existing, "node-fit", "bound-port-machine")).To(Succeed()) + + pendingName := types.NamespacedName{Name: "pending-port", Namespace: "default"} + pending := testSmolVM(pendingName, "") + pending.Spec.NodeSelector = map[string]string{"kubernetes.io/hostname": "node-fit"} + pending.Spec.Network.Ports = []vmv1alpha1.SmolVMPort{{HostPort: 8080, GuestPort: 8080}} + Expect(k8sClient.Create(ctx, pending)).To(Succeed()) + _, err := clusterReconciler().Reconcile(ctx, reconcile.Request{NamespacedName: pendingName}) + Expect(err).NotTo(HaveOccurred()) + _, err = clusterReconciler().Reconcile(ctx, reconcile.Request{NamespacedName: pendingName}) + Expect(err).NotTo(HaveOccurred()) + + latest := &vmv1alpha1.SmolVM{} + Expect(k8sClient.Get(ctx, pendingName, latest)).To(Succeed()) + Expect(latest.Status.NodeName).To(BeEmpty()) + Expect(conditionReason(latest, vmv1alpha1.ConditionScheduled)).To(Equal("NoEligibleNodes")) + Expect(conditionMessage(latest, vmv1alpha1.ConditionScheduled)).To(ContainSubstring("hostPort 8080 conflicts")) + }) + + It("rejects stale or mismatched runtime endpoints before lifecycle operations", func() { + ctx := context.Background() + runtime := newRuntimeHTTPServer("wrong-node", "uid-mismatch") + defer runtime.Close() + Expect(createReadyNode(ctx, "node-mismatch", "uid-mismatch")).To(Succeed()) + Expect(createReadySmolVMNode(ctx, "node-mismatch", "uid-mismatch", runtime.host(), runtime.port(), vmv1alpha1.SmolVMNodeAllocatable{CPUs: 4, MemoryMiB: 4096, StorageGiB: 100})).To(Succeed()) + + name := types.NamespacedName{Name: "runtime-identity-mismatch", Namespace: "default"} + Expect(createSmolVMWithStatus(ctx, name, "node-mismatch", "runtime-identity-mismatch-machine")).To(Succeed()) + _, err := clusterReconciler().Reconcile(ctx, reconcile.Request{NamespacedName: name}) + Expect(err).NotTo(HaveOccurred()) + + latest := &vmv1alpha1.SmolVM{} + Expect(k8sClient.Get(ctx, name, latest)).To(Succeed()) + Expect(conditionReason(latest, vmv1alpha1.ConditionReconciled)).To(Equal("RuntimeUnavailable")) + Expect(runtime.created).To(Equal(0)) + }) + + It("blocks deletion when the owning runtime is unavailable unless force-delete is explicit", func() { + ctx := context.Background() + blockedName := types.NamespacedName{Name: "delete-blocked-unavailable", Namespace: "default"} + Expect(createSmolVMWithStatus(ctx, blockedName, "node-gone", "delete-blocked-machine")).To(Succeed()) + Expect(k8sClient.Delete(ctx, &vmv1alpha1.SmolVM{ObjectMeta: metav1.ObjectMeta{Name: blockedName.Name, Namespace: blockedName.Namespace}})).To(Succeed()) + _, err := clusterReconciler().Reconcile(ctx, reconcile.Request{NamespacedName: blockedName}) + Expect(err).NotTo(HaveOccurred()) + blocked := &vmv1alpha1.SmolVM{} + Expect(k8sClient.Get(ctx, blockedName, blocked)).To(Succeed()) + Expect(controllerutil.ContainsFinalizer(blocked, vmv1alpha1.SmolVMFinalizer)).To(BeTrue()) + Expect(conditionReason(blocked, vmv1alpha1.ConditionDeletionBlocked)).To(Equal("RuntimeUnavailable")) + + forceName := types.NamespacedName{Name: "delete-force-unavailable", Namespace: "default"} + force := testSmolVM(forceName, "") + force.Annotations = map[string]string{vmv1alpha1.ForceDeleteLocalStateAnnotation: "true"} + Expect(createSmolVMWithSpecStatus(ctx, force, "node-gone", "delete-force-machine")).To(Succeed()) + Expect(k8sClient.Delete(ctx, &vmv1alpha1.SmolVM{ObjectMeta: metav1.ObjectMeta{Name: forceName.Name, Namespace: forceName.Namespace}})).To(Succeed()) + _, err = clusterReconciler().Reconcile(ctx, reconcile.Request{NamespacedName: forceName}) + Expect(err).NotTo(HaveOccurred()) + Eventually(func() bool { + err := k8sClient.Get(ctx, forceName, &vmv1alpha1.SmolVM{}) + return apierrors.IsNotFound(err) + }).Should(BeTrue()) + }) + It("creates a missing runtime machine and records status", func() { ctx := context.Background() name := types.NamespacedName{Name: "runtime-create", Namespace: "default"} - resource := testSmolVM(name, "") - Expect(k8sClient.Create(ctx, resource)).To(Succeed()) + Expect(createSmolVMWithStatus(ctx, name, "node-a", "runtime-create-machine")).To(Succeed()) runtime := &fakeRuntime{} reconciler := testReconciler(runtime) _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: name}) Expect(err).NotTo(HaveOccurred()) - _, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: name}) - Expect(err).NotTo(HaveOccurred()) Expect(runtime.created).To(Equal(1)) latest := &vmv1alpha1.SmolVM{} @@ -301,11 +402,9 @@ var _ = Describe("SmolVM Controller", func() { for _, tc := range cases { name := types.NamespacedName{Name: tc.name, Namespace: "default"} if tc.name == "runtime-create-error" { - Expect(k8sClient.Create(ctx, tc.vm)).To(Succeed()) + Expect(createSmolVMWithSpecStatus(ctx, tc.vm, "node-a", tc.name+"-machine")).To(Succeed()) _, err := testReconciler(tc.rt).Reconcile(ctx, reconcile.Request{NamespacedName: name}) Expect(err).NotTo(HaveOccurred()) - _, err = testReconciler(tc.rt).Reconcile(ctx, reconcile.Request{NamespacedName: name}) - Expect(err).NotTo(HaveOccurred()) } else { Expect(createSmolVMWithSpecStatus(ctx, tc.vm, "node-a", tc.name+"-machine")).To(Succeed()) _, err := testReconciler(tc.rt).Reconcile(ctx, reconcile.Request{NamespacedName: name}) @@ -356,26 +455,16 @@ var _ = Describe("SmolVM Controller", func() { Expect(latest.ResourceVersion).To(Equal(resourceVersion)) }) - It("no-ops on other nodes and reports awaiting node assignment", func() { + It("reconciles bound VMs centrally regardless of local node environment", func() { ctx := context.Background() - mismatchName := types.NamespacedName{Name: "runtime-node-mismatch", Namespace: "default"} - Expect(createSmolVMWithStatus(ctx, mismatchName, "node-a", "runtime-node-mismatch-machine")).To(Succeed()) + name := types.NamespacedName{Name: "runtime-central-bound", Namespace: "default"} + Expect(createSmolVMWithStatus(ctx, name, "node-a", "runtime-central-bound-machine")).To(Succeed()) withNodeName("node-b", func() { runtime := &fakeRuntime{} - _, err := testReconciler(runtime).Reconcile(ctx, reconcile.Request{NamespacedName: mismatchName}) - Expect(err).NotTo(HaveOccurred()) - Expect(runtime.created + runtime.started + runtime.deleted).To(Equal(0)) - }) - - pendingName := types.NamespacedName{Name: "runtime-awaiting-node", Namespace: "default"} - Expect(k8sClient.Create(ctx, testSmolVM(pendingName, ""))).To(Succeed()) - withNodeName("node-a", func() { - _, err := testReconciler(&fakeRuntime{}).Reconcile(ctx, reconcile.Request{NamespacedName: pendingName}) + _, err := testReconciler(runtime).Reconcile(ctx, reconcile.Request{NamespacedName: name}) Expect(err).NotTo(HaveOccurred()) + Expect(runtime.created).To(Equal(1)) }) - latest := &vmv1alpha1.SmolVM{} - Expect(k8sClient.Get(ctx, pendingName, latest)).To(Succeed()) - Expect(conditionReason(latest, vmv1alpha1.ConditionReconciled)).To(Equal("AwaitingNodeAssignment")) }) It("rejects invalid specs through CRD admission", func() { @@ -396,13 +485,11 @@ var _ = Describe("SmolVM Controller", func() { It("emits Kubernetes events for lifecycle and failure transitions", func() { ctx := context.Background() createName := types.NamespacedName{Name: "event-create", Namespace: "default"} - Expect(k8sClient.Create(ctx, testSmolVM(createName, "node-a"))).To(Succeed()) + Expect(createSmolVMWithStatus(ctx, createName, "node-a", "event-create-machine")).To(Succeed()) createRecorder := record.NewFakeRecorder(4) createRuntime := &fakeRuntime{} _, err := testReconcilerWithRecorder(createRuntime, createRecorder).Reconcile(ctx, reconcile.Request{NamespacedName: createName}) Expect(err).NotTo(HaveOccurred()) - _, err = testReconcilerWithRecorder(createRuntime, createRecorder).Reconcile(ctx, reconcile.Request{NamespacedName: createName}) - Expect(err).NotTo(HaveOccurred()) Eventually(createRecorder.Events).Should(Receive(ContainSubstring("Creating"))) failureName := types.NamespacedName{Name: "event-runtime-failure", Namespace: "default"} @@ -461,6 +548,12 @@ func testReconcilerWithRecorder(runtime *fakeRuntime, recorder record.EventRecor } } +func clusterReconciler() *SmolVMReconciler { + Expect(os.Setenv("SMOLVM_RUNTIME_TOKEN", "test-token")).To(Succeed()) + Expect(os.Setenv("SMOLVM_RUNTIME_INSECURE_SKIP_VERIFY", "true")).To(Succeed()) + return &SmolVMReconciler{Client: k8sClient, Scheme: k8sClient.Scheme()} +} + func createSmolVMWithStatus(ctx context.Context, name types.NamespacedName, nodeName, machineName string) error { return createSmolVMWithSpecStatus(ctx, testSmolVM(name, nodeName), nodeName, machineName) } @@ -502,6 +595,130 @@ func conditionReason(vm *vmv1alpha1.SmolVM, conditionType string) string { return "" } +func conditionMessage(vm *vmv1alpha1.SmolVM, conditionType string) string { + for _, condition := range vm.Status.Conditions { + if condition.Type == conditionType { + return condition.Message + } + } + return "" +} + +func createReadyNode(ctx context.Context, name, uid string) error { + node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: name, UID: types.UID(uid), Labels: map[string]string{"kubernetes.io/arch": "amd64"}}} + if err := k8sClient.Create(ctx, node); err != nil && !apierrors.IsAlreadyExists(err) { + return err + } + latest := &corev1.Node{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: name}, latest); err != nil { + return err + } + latest.Status.Conditions = []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}} + latest.Status.Allocatable = corev1.ResourceList{} + return k8sClient.Status().Update(ctx, latest) +} + +func createReadySmolVMNode(ctx context.Context, name, uid, host string, port int32, allocatable vmv1alpha1.SmolVMNodeAllocatable) error { + node := &vmv1alpha1.SmolVMNode{ObjectMeta: metav1.ObjectMeta{Name: name, Labels: map[string]string{"kubernetes.io/arch": "amd64", "kubernetes.io/hostname": name}}} + if err := k8sClient.Create(ctx, node); err != nil && !apierrors.IsAlreadyExists(err) { + return err + } + latest := &vmv1alpha1.SmolVMNode{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: name}, latest); err != nil { + return err + } + now := metav1.NewTime(time.Now()) + latest.Status = vmv1alpha1.SmolVMNodeStatus{ + NodeUID: uid, + RuntimeVersion: "test", + ProtocolVersion: "v1alpha1", + HeartbeatTime: &now, + Endpoint: vmv1alpha1.SmolVMNodeEndpoint{PodIP: host, Port: port}, + Allocatable: allocatable, + Conditions: []metav1.Condition{ + {Type: vmv1alpha1.SmolVMNodeConditionReady, Status: metav1.ConditionTrue, Reason: "Test", Message: "ready", LastTransitionTime: now}, + {Type: vmv1alpha1.SmolVMNodeConditionKVMAvailable, Status: metav1.ConditionTrue, Reason: "Test", Message: "kvm", LastTransitionTime: now}, + {Type: vmv1alpha1.SmolVMNodeConditionRuntimeReady, Status: metav1.ConditionTrue, Reason: "Test", Message: "runtime", LastTransitionTime: now}, + {Type: vmv1alpha1.SmolVMNodeConditionSchedulable, Status: metav1.ConditionTrue, Reason: "Test", Message: "schedulable", LastTransitionTime: now}, + }, + } + return k8sClient.Status().Update(ctx, latest) +} + +type runtimeHTTPServer struct { + *httptest.Server + nodeName string + nodeUID string + created int +} + +func newRuntimeHTTPServer(nodeName, nodeUID string) *runtimeHTTPServer { + runtime := &runtimeHTTPServer{nodeName: nodeName, nodeUID: nodeUID} + mux := http.NewServeMux() + mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer test-token" { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + _ = json.NewEncoder(w).Encode(smolvmapi.Health{OK: true, KVMAvailable: true, SocketReady: true, StateReady: true}) + }) + mux.HandleFunc("/api/v1/identity", func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer test-token" { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + _ = json.NewEncoder(w).Encode(smolvmapi.Identity{NodeName: runtime.nodeName, NodeUID: runtime.nodeUID}) + }) + mux.HandleFunc("/api/v1/capabilities", func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer test-token" { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + _ = json.NewEncoder(w).Encode(smolvmapi.Capabilities{ProtocolVersion: "v1alpha1", KVMAvailable: true}) + }) + mux.HandleFunc("/api/v1/machines", func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer test-token" { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + if r.Method != http.MethodPost { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + runtime.created++ + var req smolvmapi.CreateMachineRequest + Expect(json.NewDecoder(r.Body).Decode(&req)).To(Succeed()) + _ = json.NewEncoder(w).Encode(smolvmapi.MachineInfo{Name: req.Name, State: "stopped", CPUs: req.CPUs, MemoryMiB: req.MemoryMiB}) + }) + mux.HandleFunc("/api/v1/machines/", func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer test-token" { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + http.NotFound(w, r) + }) + runtime.Server = httptest.NewTLSServer(mux) + return runtime +} + +func (s *runtimeHTTPServer) host() string { + parsed, err := url.Parse(s.URL) + Expect(err).NotTo(HaveOccurred()) + host, _, err := net.SplitHostPort(parsed.Host) + Expect(err).NotTo(HaveOccurred()) + return host +} + +func (s *runtimeHTTPServer) port() int32 { + parsed, err := url.Parse(s.URL) + Expect(err).NotTo(HaveOccurred()) + _, portString, err := net.SplitHostPort(parsed.Host) + Expect(err).NotTo(HaveOccurred()) + port, err := strconv.Atoi(portString) + Expect(err).NotTo(HaveOccurred()) + return int32(port) +} + type fakeRuntime struct { machine *smolvmapi.MachineInfo getErr error @@ -517,6 +734,25 @@ type fakeRuntime struct { resized int } +func (f *fakeRuntime) Health(context.Context) (*smolvmapi.Health, error) { + return &smolvmapi.Health{OK: true, KVMAvailable: true, SocketReady: true, StateReady: true}, nil +} + +func (f *fakeRuntime) GetIdentity(context.Context) (*smolvmapi.Identity, error) { + return &smolvmapi.Identity{NodeName: "node-a", NodeUID: "uid-a"}, nil +} + +func (f *fakeRuntime) Capabilities(context.Context) (*smolvmapi.Capabilities, error) { + return &smolvmapi.Capabilities{ProtocolVersion: "v1alpha1", KVMAvailable: true}, nil +} + +func (f *fakeRuntime) ListMachines(context.Context) ([]smolvmapi.MachineInfo, error) { + if f.machine == nil { + return nil, nil + } + return []smolvmapi.MachineInfo{*f.machine}, nil +} + func (f *fakeRuntime) GetMachine(context.Context, string) (*smolvmapi.MachineInfo, error) { if f.getErr != nil { return nil, f.getErr diff --git a/internal/smolvm/client.go b/internal/smolvm/client.go index 833dc33..22ed3b9 100644 --- a/internal/smolvm/client.go +++ b/internal/smolvm/client.go @@ -3,6 +3,7 @@ package smolvm import ( "bytes" "context" + "crypto/tls" "encoding/json" "fmt" "io" @@ -17,15 +18,24 @@ import ( type Client struct { baseURL string http *http.Client + token string } // NewClient creates a smolvm API client. If socketPath is non-empty, baseURL is // used only for URL construction and requests are transported over the socket. func NewClient(baseURL, socketPath string) *Client { + return NewClientWithAuth(baseURL, socketPath, "", false) +} + +// NewClientWithAuth creates a smolvm API client with optional bearer-token auth. +func NewClientWithAuth(baseURL, socketPath, token string, insecureSkipVerify bool) *Client { if baseURL == "" { baseURL = "http://127.0.0.1:8080" } transport := http.DefaultTransport.(*http.Transport).Clone() + if insecureSkipVerify { + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint:gosec + } if socketPath != "" { transport.DialContext = func(ctx context.Context, _, _ string) (net.Conn, error) { return (&net.Dialer{}).DialContext(ctx, "unix", socketPath) @@ -34,6 +44,7 @@ func NewClient(baseURL, socketPath string) *Client { } return &Client{ baseURL: strings.TrimRight(baseURL, "/"), + token: token, http: &http.Client{ Timeout: 30 * time.Second, Transport: transport, @@ -58,6 +69,31 @@ type ListMachinesResponse struct { Machines []MachineInfo `json:"machines"` } +// Identity reports the runtime API node identity. +type Identity struct { + NodeName string `json:"nodeName"` + NodeUID string `json:"nodeUID,omitempty"` +} + +// Health reports runtime health. +type Health struct { + OK bool `json:"ok"` + KVMAvailable bool `json:"kvmAvailable"` + SocketReady bool `json:"socketReady"` + StateReady bool `json:"stateReady"` + Message string `json:"message,omitempty"` +} + +// Capabilities reports runtime node resources and features. +type Capabilities struct { + RuntimeVersion string `json:"runtimeVersion,omitempty"` + ProtocolVersion string `json:"protocolVersion,omitempty"` + CPUs int32 `json:"cpus,omitempty"` + MemoryMiB int64 `json:"memoryMiB,omitempty"` + StorageGiB int64 `json:"storageGiB,omitempty"` + KVMAvailable bool `json:"kvmAvailable"` +} + type CreateMachineRequest struct { Name string `json:"name,omitempty"` CPUs int32 `json:"cpus,omitempty"` @@ -91,6 +127,38 @@ type ExecResponse struct { Stderr string `json:"stderr"` } +func (c *Client) Health(ctx context.Context) (*Health, error) { + var health Health + if err := c.request(ctx, http.MethodGet, "/healthz", nil, &health); err != nil { + return nil, err + } + return &health, nil +} + +func (c *Client) GetIdentity(ctx context.Context) (*Identity, error) { + var identity Identity + if err := c.request(ctx, http.MethodGet, "/api/v1/identity", nil, &identity); err != nil { + return nil, err + } + return &identity, nil +} + +func (c *Client) Capabilities(ctx context.Context) (*Capabilities, error) { + var capabilities Capabilities + if err := c.request(ctx, http.MethodGet, "/api/v1/capabilities", nil, &capabilities); err != nil { + return nil, err + } + return &capabilities, nil +} + +func (c *Client) ListMachines(ctx context.Context) ([]MachineInfo, error) { + var result ListMachinesResponse + if err := c.request(ctx, http.MethodGet, "/api/v1/machines", nil, &result); err != nil { + return nil, err + } + return result.Machines, nil +} + func (c *Client) GetMachine(ctx context.Context, name string) (*MachineInfo, error) { var machine MachineInfo if err := c.request(ctx, http.MethodGet, "/api/v1/machines/"+url.PathEscape(name), nil, &machine); err != nil { @@ -155,6 +223,9 @@ func (c *Client) request(ctx context.Context, method, path string, in, out any) if in != nil { req.Header.Set("Content-Type", "application/json") } + if c.token != "" { + req.Header.Set("Authorization", "Bearer "+c.token) + } resp, err := c.http.Do(req) if err != nil { diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index ee72ae2..c114f06 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -24,7 +24,7 @@ var _ = Describe("controller", Ordered, func() { _, _ = utils.Run(exec.Command("kubectl", "delete", "-k", "config/crd", "--ignore-not-found=true", "--wait=false")) }) - It("deploys on kind and reports RuntimeUnavailable without smolvm", func() { + It("deploys the controller/runtime topology and reports no eligible nodes without KVM", func() { By("building the manager image") _, err := utils.Run(exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectImage))) Expect(err).NotTo(HaveOccurred()) @@ -36,24 +36,31 @@ var _ = Describe("controller", Ordered, func() { _, err = utils.Run(exec.Command("make", "install")) Expect(err).NotTo(HaveOccurred()) - By("deploying the controller") - _, err = utils.Run(exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectImage))) + By("deploying the controller and runtime") + _, err = utils.Run(exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectImage), fmt.Sprintf("RUNTIME_IMG=%s", projectImage))) Expect(err).NotTo(HaveOccurred()) - By("waiting for the controller daemonset") - _, err = utils.Run(exec.Command("kubectl", "rollout", "status", "daemonset/operator-controller-manager", "-n", namespace, "--timeout=2m")) + By("waiting for the controller deployment") + _, err = utils.Run(exec.Command("kubectl", "rollout", "status", "deployment/operator-controller-manager", "-n", namespace, "--timeout=2m")) Expect(err).NotTo(HaveOccurred()) - By("applying a node-pinned SmolVM resource") + By("waiting for the runtime daemonset pod") + Eventually(func() string { + out, err := utils.Run(exec.Command("kubectl", "get", "daemonset/operator-smolvm-runtime", "-n", namespace, "-o", "jsonpath={.status.desiredNumberScheduled}")) + if err != nil { + return err.Error() + } + return strings.TrimSpace(string(out)) + }, 2*time.Minute, 2*time.Second).ShouldNot(Equal("0")) + + By("applying an unpinned SmolVM resource") _, err = utils.Run(exec.Command("bash", "-c", ` -node=$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}') cat < Date: Wed, 3 Jun 2026 10:41:43 +0200 Subject: [PATCH 2/2] Move runtime lifecycle e2e to Ginkgo --- .github/workflows/ci.yml | 134 +------------------- Makefile | 5 + test/e2e/runtime_lifecycle_test.go | 197 +++++++++++++++++++++++++++++ 3 files changed, 208 insertions(+), 128 deletions(-) create mode 100644 test/e2e/runtime_lifecycle_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f314ad6..436e3cd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,134 +70,12 @@ jobs: with: cluster_name: runtime config: .github/kind-runtime-config.yaml - - name: Build and load operator images - run: | - set -euxo pipefail - make docker-build IMG=example.com/smolvm-operator:e2e - make docker-build-runtime RUNTIME_IMG=example.com/smolvm-runtime:e2e - docker run --rm --entrypoint smolvm example.com/smolvm-runtime:e2e --version - docker run --rm --entrypoint /runtime-agent example.com/smolvm-runtime:e2e --help - kind load docker-image example.com/smolvm-operator:e2e --name runtime - kind load docker-image example.com/smolvm-runtime:e2e --name runtime - - name: Deploy full stack and run VM lifecycle - run: | - set -euxo pipefail - mkdir -p reports - current_step="install" - write_runtime_report() { - local status="$1" - local failed_step="${2:-}" - if [ "$status" = "pass" ]; then - cat > reports/runtime.xml <<'EOF' - - - - - - - - - - - - EOF - return - fi - cat > reports/runtime.xml < - - - - - EOF - } - cleanup() { - status=$? - if [ ! -f reports/runtime.xml ]; then - if [ "$status" -eq 0 ]; then - write_runtime_report pass - else - write_runtime_report fail "$current_step" - fi - fi - kubectl delete smolvm runtime-smoke --ignore-not-found=true --wait=false || true - kubectl -n operator-system get pods -o wide || true - kubectl -n operator-system logs deployment/operator-controller-manager --all-containers=true || true - kubectl -n operator-system logs daemonset/operator-smolvm-runtime --all-containers=true || true - exit "$status" - } - trap cleanup EXIT - - current_step="install CRDs" - make install - current_step="deploy full operator stack" - make deploy IMG=example.com/smolvm-operator:e2e RUNTIME_IMG=example.com/smolvm-runtime:e2e - - current_step="controller Deployment rollout" - kubectl rollout status deployment/operator-controller-manager -n operator-system --timeout=3m - current_step="runtime DaemonSet rollout" - kubectl rollout status daemonset/operator-smolvm-runtime -n operator-system --timeout=5m - - current_step="SmolVMNode readiness" - node=$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}') - for i in {1..90}; do - kubectl get smolvmnode "$node" -o yaml || true - ready=$(kubectl get smolvmnode "$node" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true) - runtime_ready=$(kubectl get smolvmnode "$node" -o jsonpath='{.status.conditions[?(@.type=="RuntimeReady")].status}' 2>/dev/null || true) - if [ "$ready" = "True" ] && [ "$runtime_ready" = "True" ]; then - break - fi - sleep 2 - done - test "$(kubectl get smolvmnode "$node" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}')" = "True" - test "$(kubectl get smolvmnode "$node" -o jsonpath='{.status.conditions[?(@.type=="RuntimeReady")].status}')" = "True" - test "$(kubectl get smolvmnode "$node" -o jsonpath='{.status.endpoint.podNamespace}')" = "operator-system" - - current_step="create unpinned SmolVM" - cat <<'EOF' | kubectl apply -f - - apiVersion: vm.smolvm.dev/v1alpha1 - kind: SmolVM - metadata: - name: runtime-smoke - spec: - running: true - image: alpine:latest - resources: - cpus: 1 - memoryMiB: 128 - EOF - - current_step="SmolVM reaches Running" - for i in {1..60}; do - kubectl get smolvm runtime-smoke -o yaml || true - status=$(kubectl get smolvm runtime-smoke -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true) - if [ "$status" = "True" ]; then - break - fi - sleep 10 - done - test "$(kubectl get smolvm runtime-smoke -o jsonpath='{.status.conditions[?(@.type=="Scheduled")].reason}')" = "Bound" - test "$(kubectl get smolvm runtime-smoke -o jsonpath='{.status.conditions[?(@.type=="RuntimeReady")].status}')" = "True" - test "$(kubectl get smolvm runtime-smoke -o jsonpath='{.status.phase}')" = "Running" - - current_step="patch spec.running false" - kubectl patch smolvm runtime-smoke --type=merge -p '{"spec":{"running":false}}' - current_step="SmolVM reaches Stopped" - kubectl wait smolvm runtime-smoke --for=jsonpath='{.status.phase}'=Stopped --timeout=5m - - current_step="delete CR cleans runtime machine" - machine=$(kubectl get smolvm runtime-smoke -o jsonpath='{.status.machineName}') - runtime_pod=$(kubectl -n operator-system get pod -l app.kubernetes.io/name=smolvm-runtime -o jsonpath='{.items[0].metadata.name}') - kubectl delete smolvm runtime-smoke --wait=true --timeout=5m - for i in {1..30}; do - if ! kubectl -n operator-system exec "$runtime_pod" -- smolvm machine ls | grep -q "$machine"; then - exit 0 - fi - sleep 2 - done - echo "runtime machine $machine still exists after CR deletion" - kubectl -n operator-system exec "$runtime_pod" -- smolvm machine ls || true - exit 1 + - name: Run runtime lifecycle e2e + env: + RUN_E2E: "true" + KIND_CLUSTER: runtime + RUNTIME_E2E: "true" + run: make test-runtime-e2e-report - name: Upload runtime logs if: always() uses: actions/upload-artifact@v4 diff --git a/Makefile b/Makefile index 07bbfa6..0967555 100644 --- a/Makefile +++ b/Makefile @@ -81,6 +81,11 @@ test-e2e-report: gotestsum ## Run e2e tests and write a JUnit report under repor mkdir -p reports $(GOTESTSUM) --format testname --junitfile reports/e2e.xml -- ./test/e2e/ -v -ginkgo.v +.PHONY: test-runtime-e2e-report +test-runtime-e2e-report: ## Run runtime lifecycle e2e tests and write a Ginkgo JUnit report under reports/. + mkdir -p reports + go test ./test/e2e -v -ginkgo.v -ginkgo.label-filter=runtime -ginkgo.junit-report=reports/runtime.xml + .PHONY: lint lint: golangci-lint ## Run golangci-lint linter & yamllint $(GOLANGCI_LINT) run diff --git a/test/e2e/runtime_lifecycle_test.go b/test/e2e/runtime_lifecycle_test.go new file mode 100644 index 0000000..9441a5d --- /dev/null +++ b/test/e2e/runtime_lifecycle_test.go @@ -0,0 +1,197 @@ +package e2e + +import ( + "fmt" + "os" + "os/exec" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/manuschillerdev/smolvm-operator/test/utils" +) + +const ( + runtimeControllerImage = "example.com/smolvm-operator:e2e" + runtimeImage = "example.com/smolvm-runtime:e2e" + runtimeSmolVMName = "runtime-smoke" +) + +var _ = Describe("runtime lifecycle", Label("runtime"), Ordered, func() { + var machineName string + + BeforeAll(func() { + if os.Getenv("RUNTIME_E2E") != "true" { + Skip("set RUNTIME_E2E=true to run runtime lifecycle e2e tests") + } + }) + + AfterEach(func() { + if CurrentSpecReport().Failed() { + dumpRuntimeDiagnostics() + } + }) + + AfterAll(func() { + if os.Getenv("RUNTIME_E2E") != "true" { + return + } + + _, _ = utils.Run(exec.Command("kubectl", "delete", "smolvm", runtimeSmolVMName, "--ignore-not-found=true", "--wait=false")) + _, _ = utils.Run(exec.Command("kubectl", "delete", "-k", "config/default", "--ignore-not-found=true", "--wait=false")) + _, _ = utils.Run(exec.Command("kubectl", "delete", "-k", "config/crd", "--ignore-not-found=true", "--wait=false")) + }) + + It("builds and sanity-checks controller and runtime images", func() { + By("building the controller image") + _, err := utils.Run(exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", runtimeControllerImage))) + Expect(err).NotTo(HaveOccurred()) + + By("building the runtime image") + _, err = utils.Run(exec.Command("make", "docker-build-runtime", fmt.Sprintf("RUNTIME_IMG=%s", runtimeImage))) + Expect(err).NotTo(HaveOccurred()) + + By("checking the smolvm binary") + _, err = utils.Run(exec.Command("docker", "run", "--rm", "--entrypoint", "smolvm", runtimeImage, "--version")) + Expect(err).NotTo(HaveOccurred()) + + By("checking the runtime agent binary") + _, err = utils.Run(exec.Command("docker", "run", "--rm", "--entrypoint", "/runtime-agent", runtimeImage, "--help")) + Expect(err).NotTo(HaveOccurred()) + }) + + It("loads images into kind", func() { + Expect(utils.LoadImageToKindClusterWithName(runtimeControllerImage)).To(Succeed()) + Expect(utils.LoadImageToKindClusterWithName(runtimeImage)).To(Succeed()) + }) + + It("installs CRDs and deploys the full stack", func() { + By("installing CRDs") + _, err := utils.Run(exec.Command("make", "install")) + Expect(err).NotTo(HaveOccurred()) + + By("deploying controller and runtime") + _, err = utils.Run(exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", runtimeControllerImage), fmt.Sprintf("RUNTIME_IMG=%s", runtimeImage))) + Expect(err).NotTo(HaveOccurred()) + }) + + It("waits for controller Deployment and runtime DaemonSet", func() { + _, err := utils.Run(exec.Command("kubectl", "rollout", "status", "deployment/operator-controller-manager", "-n", namespace, "--timeout=3m")) + Expect(err).NotTo(HaveOccurred()) + + _, err = utils.Run(exec.Command("kubectl", "rollout", "status", "daemonset/operator-smolvm-runtime", "-n", namespace, "--timeout=5m")) + Expect(err).NotTo(HaveOccurred()) + }) + + It("waits for SmolVMNode Ready and RuntimeReady", func() { + nodeName := firstNodeName() + Expect(nodeName).NotTo(BeEmpty()) + + Eventually(func(g Gomega) { + out, err := utils.Run(exec.Command("kubectl", "get", "smolvmnode", nodeName, "-o", "yaml")) + if err == nil { + fmt.Fprintln(GinkgoWriter, string(out)) + } + + ready, err := kubectlOutputE("get", "smolvmnode", nodeName, "-o", `jsonpath={.status.conditions[?(@.type=="Ready")].status}`) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(ready).To(Equal("True")) + + runtimeReady, err := kubectlOutputE("get", "smolvmnode", nodeName, "-o", `jsonpath={.status.conditions[?(@.type=="RuntimeReady")].status}`) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(runtimeReady).To(Equal("True")) + }, 3*time.Minute, 2*time.Second).Should(Succeed()) + + Expect(kubectlOutput("get", "smolvmnode", nodeName, "-o", "jsonpath={.status.endpoint.podNamespace}")).To(Equal(namespace)) + }) + + It("creates an unpinned SmolVM and observes Running", func() { + cmd := exec.Command("kubectl", "apply", "-f", "-") + cmd.Stdin = strings.NewReader(`apiVersion: vm.smolvm.dev/v1alpha1 +kind: SmolVM +metadata: + name: runtime-smoke +spec: + running: true + image: alpine:latest + resources: + cpus: 1 + memoryMiB: 128 +`) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func() string { + out, err := utils.Run(exec.Command("kubectl", "get", "smolvm", runtimeSmolVMName, "-o", `jsonpath={.status.conditions[?(@.type=="Ready")].status}`)) + if err != nil { + return err.Error() + } + return strings.TrimSpace(string(out)) + }, 10*time.Minute, 10*time.Second).Should(Equal("True")) + + Expect(kubectlOutput("get", "smolvm", runtimeSmolVMName, "-o", `jsonpath={.status.conditions[?(@.type=="Scheduled")].reason}`)).To(Equal("Bound")) + Expect(kubectlOutput("get", "smolvm", runtimeSmolVMName, "-o", `jsonpath={.status.conditions[?(@.type=="RuntimeReady")].status}`)).To(Equal("True")) + Expect(kubectlOutput("get", "smolvm", runtimeSmolVMName, "-o", "jsonpath={.status.phase}")).To(Equal("Running")) + }) + + It("patches spec.running=false and observes Stopped", func() { + _, err := utils.Run(exec.Command("kubectl", "patch", "smolvm", runtimeSmolVMName, "--type=merge", "-p", `{"spec":{"running":false}}`)) + Expect(err).NotTo(HaveOccurred()) + + _, err = utils.Run(exec.Command("kubectl", "wait", "smolvm", runtimeSmolVMName, "--for=jsonpath={.status.phase}=Stopped", "--timeout=5m")) + Expect(err).NotTo(HaveOccurred()) + }) + + It("deletes the SmolVM and verifies runtime machine cleanup", func() { + machineName = kubectlOutput("get", "smolvm", runtimeSmolVMName, "-o", "jsonpath={.status.machineName}") + Expect(machineName).NotTo(BeEmpty()) + + runtimePod := kubectlOutput("-n", namespace, "get", "pod", "-l", "app.kubernetes.io/name=smolvm-runtime", "-o", "jsonpath={.items[0].metadata.name}") + Expect(runtimePod).NotTo(BeEmpty()) + + _, err := utils.Run(exec.Command("kubectl", "delete", "smolvm", runtimeSmolVMName, "--wait=true", "--timeout=5m")) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func() string { + out, err := utils.Run(exec.Command("kubectl", "-n", namespace, "exec", runtimePod, "--", "smolvm", "machine", "ls")) + if err != nil { + return err.Error() + } + return string(out) + }, time.Minute, 2*time.Second).ShouldNot(ContainSubstring(machineName)) + }) +}) + +func firstNodeName() string { + return kubectlOutput("get", "nodes", "-o", "jsonpath={.items[0].metadata.name}") +} + +func kubectlOutput(args ...string) string { + out, err := kubectlOutputE(args...) + Expect(err).NotTo(HaveOccurred()) + return out +} + +func kubectlOutputE(args ...string) (string, error) { + out, err := utils.Run(exec.Command("kubectl", args...)) + return strings.TrimSpace(string(out)), err +} + +func dumpRuntimeDiagnostics() { + commands := []*exec.Cmd{ + exec.Command("kubectl", "-n", namespace, "get", "pods", "-o", "wide"), + exec.Command("kubectl", "-n", namespace, "logs", "deployment/operator-controller-manager", "--all-containers=true"), + exec.Command("kubectl", "-n", namespace, "logs", "daemonset/operator-smolvm-runtime", "--all-containers=true"), + exec.Command("kubectl", "get", "smolvmnodes", "-o", "yaml"), + exec.Command("kubectl", "get", "smolvm", runtimeSmolVMName, "-o", "yaml"), + } + for _, cmd := range commands { + out, err := utils.Run(cmd) + fmt.Fprintf(GinkgoWriter, "\n--- %s ---\n%s", strings.Join(cmd.Args, " "), string(out)) + if err != nil { + fmt.Fprintf(GinkgoWriter, "warning: %v\n", err) + } + } +}