Implement operator-managed runtime architecture

Author: manuschillerdev

.github/kind-runtime-config.yaml

@@ -0,0 +1,11 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+  extraMounts:
+  - hostPath: /dev/kvm
+    containerPath: /dev/kvm
+  - hostPath: /tmp/smolvm-kind-var-lib
+    containerPath: /var/lib/smolvm
+  - hostPath: /tmp/smolvm-kind-var-run
+    containerPath: /var/run/smolvm

.github/workflows/ci.yml

@@ -17,10 +17,16 @@ jobs:
           go-version-file: go.mod
           cache: true
       - name: Run controller-runtime envtest
-        run: make test
+        run: make test-report
+      - name: Upload test reports
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: envtest-reports
+          path: reports/
 
   kind-e2e:
-    name: kind e2e
+    name: kind topology e2e
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -31,55 +37,76 @@ jobs:
       - uses: helm/kind-action@v1
         with:
           cluster_name: kind
-      - name: Run one deployment e2e test
+      - name: Run deployed-topology e2e test
         env:
           RUN_E2E: "true"
           KIND_CLUSTER: kind
-        run: make test-e2e
+        run: make test-e2e-report
+      - name: Upload e2e reports
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: e2e-reports
+          path: reports/
 
-  smolvm-runtime-smoke:
-    name: smolvm runtime smoke
+  smolvm-runtime-e2e:
+    name: smolvm runtime lifecycle e2e
     runs-on: ubuntu-latest
-    timeout-minutes: 20
+    timeout-minutes: 45
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
           go-version-file: go.mod
           cache: true
+      - name: Prepare KVM-backed kind mounts
+        run: |
+          set -euxo pipefail
+          test -e /dev/kvm
+          sudo chmod 666 /dev/kvm
+          sudo mkdir -p /tmp/smolvm-kind-var-lib /tmp/smolvm-kind-var-run
+          sudo chmod 777 /tmp/smolvm-kind-var-lib /tmp/smolvm-kind-var-run
       - uses: helm/kind-action@v1
         with:
           cluster_name: runtime
-      - name: Install smolvm
+          config: .github/kind-runtime-config.yaml
+      - name: Build and load operator images
         run: |
-          sudo apt-get update
-          sudo apt-get install -y libvirglrenderer1 libepoxy0 libgbm1
-          curl -fsSL -o /tmp/smolvm.tar.gz https://github.com/smol-machines/smolvm/releases/download/v0.8.1/smolvm-0.8.1-linux-x86_64.tar.gz
-          mkdir -p /tmp/smolvm
-          tar -xzf /tmp/smolvm.tar.gz -C /tmp/smolvm --strip-components=1
-          echo /tmp/smolvm >> "$GITHUB_PATH"
-          /tmp/smolvm/smolvm --version
-      - name: Run operator against real smolvm runtime
+          set -euxo pipefail
+          make docker-build IMG=example.com/smolvm-operator:e2e
+          make docker-build-runtime RUNTIME_IMG=example.com/smolvm-runtime:e2e
+          kind load docker-image example.com/smolvm-operator:e2e --name runtime
+          kind load docker-image example.com/smolvm-runtime:e2e --name runtime
+      - name: Deploy full stack and run VM lifecycle
         run: |
           set -euxo pipefail
-          sudo chmod 666 /dev/kvm
-          export LD_LIBRARY_PATH="/tmp/smolvm/lib:${LD_LIBRARY_PATH:-}"
-          export LD_PRELOAD="$(find /usr/lib -name 'libvirglrenderer.so.1' | head -1)"
-          smolvm serve start --listen 127.0.0.1:8080 > smolvm-serve.log 2>&1 &
-          serve_pid=$!
-          trap 'kubectl delete smolvm runtime-smoke --ignore-not-found=true --wait=false || true; kill $serve_pid || true; cat smolvm-serve.log || true' EXIT
+          make install
+          make deploy IMG=example.com/smolvm-operator:e2e RUNTIME_IMG=example.com/smolvm-runtime:e2e
 
-          for i in {1..60}; do
-            curl -fsS http://127.0.0.1:8080/api/v1/machines && break
-            sleep 1
-          done
+          cleanup() {
+            kubectl delete smolvm runtime-smoke --ignore-not-found=true --wait=false || true
+            kubectl -n operator-system get pods -o wide || true
+            kubectl -n operator-system logs deployment/operator-controller-manager --all-containers=true || true
+            kubectl -n operator-system logs daemonset/operator-smolvm-runtime --all-containers=true || true
+          }
+          trap cleanup EXIT
 
-          make install
-          SMOLVM_API_URL=http://127.0.0.1:8080 go run ./cmd/main.go \
-            --metrics-bind-address=0 \
-            --health-probe-bind-address=:8081 > manager.log 2>&1 &
-          manager_pid=$!
-          trap 'kubectl delete smolvm runtime-smoke --ignore-not-found=true --wait=false || true; kill $manager_pid || true; kill $serve_pid || true; cat manager.log || true; cat smolvm-serve.log || true' EXIT
+          kubectl rollout status deployment/operator-controller-manager -n operator-system --timeout=3m
+          kubectl rollout status daemonset/operator-smolvm-runtime -n operator-system --timeout=5m
+
+          node=$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}')
+          for i in {1..90}; do
+            kubectl get smolvmnode "$node" -o yaml || true
+            ready=$(kubectl get smolvmnode "$node" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)
+            runtime_ready=$(kubectl get smolvmnode "$node" -o jsonpath='{.status.conditions[?(@.type=="RuntimeReady")].status}' 2>/dev/null || true)
+            if [ "$ready" = "True" ] && [ "$runtime_ready" = "True" ]; then
+              break
+            fi
+            sleep 2
+          done
+          test "$(kubectl get smolvmnode "$node" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}')" = "True"
+          test "$(kubectl get smolvmnode "$node" -o jsonpath='{.status.conditions[?(@.type=="RuntimeReady")].status}')" = "True"
+          test "$(kubectl get smolvmnode "$node" -o jsonpath='{.status.endpoint.podNamespace}')" = "operator-system"
 
           cat <<'EOF' | kubectl apply -f -
           apiVersion: vm.smolvm.dev/v1alpha1
@@ -94,40 +121,36 @@ jobs:
               memoryMiB: 128
           EOF
 
-          ready=false
-          for i in {1..48}; do
-            echo "--- poll $i ---"
+          for i in {1..60}; do
             kubectl get smolvm runtime-smoke -o yaml || true
-            curl -fsS http://127.0.0.1:8080/api/v1/machines || true
             status=$(kubectl get smolvm runtime-smoke -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)
             if [ "$status" = "True" ]; then
-              ready=true
               break
             fi
             sleep 10
           done
-          if [ "$ready" != "true" ]; then
-            echo "runtime-smoke did not become Ready"
-            kubectl get smolvm runtime-smoke -o yaml || true
-            curl -fsS http://127.0.0.1:8080/api/v1/machines || true
-            exit 1
-          fi
+          test "$(kubectl get smolvm runtime-smoke -o jsonpath='{.status.conditions[?(@.type=="Scheduled")].reason}')" = "Bound"
           test "$(kubectl get smolvm runtime-smoke -o jsonpath='{.status.conditions[?(@.type=="RuntimeReady")].status}')" = "True"
-          test "$(kubectl get smolvm runtime-smoke -o jsonpath='{.status.conditions[?(@.type=="GuestReady")].status}')" = "Unknown"
+          test "$(kubectl get smolvm runtime-smoke -o jsonpath='{.status.phase}')" = "Running"
 
           kubectl patch smolvm runtime-smoke --type=merge -p '{"spec":{"running":false}}'
-          kubectl wait smolvm runtime-smoke \
-            --for=jsonpath='{.status.phase}'=Stopped \
-            --timeout=5m
+          kubectl wait smolvm runtime-smoke --for=jsonpath='{.status.phase}'=Stopped --timeout=5m
 
           machine=$(kubectl get smolvm runtime-smoke -o jsonpath='{.status.machineName}')
+          runtime_pod=$(kubectl -n operator-system get pod -l app.kubernetes.io/name=smolvm-runtime -o jsonpath='{.items[0].metadata.name}')
           kubectl delete smolvm runtime-smoke --wait=true --timeout=5m
           for i in {1..30}; do
-            if ! curl -fsS http://127.0.0.1:8080/api/v1/machines | grep -q "$machine"; then
+            if ! kubectl -n operator-system exec "$runtime_pod" -- smolvm machine ls | grep -q "$machine"; then
               exit 0
             fi
             sleep 2
           done
           echo "runtime machine $machine still exists after CR deletion"
-          curl -fsS http://127.0.0.1:8080/api/v1/machines || true
+          kubectl -n operator-system exec "$runtime_pod" -- smolvm machine ls || true
           exit 1
+      - name: Upload runtime logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: runtime-e2e-logs
+          path: reports/

.gitignore

@@ -7,6 +7,7 @@
 *.dylib
 bin/*
 Dockerfile.cross
+reports/
 
 # Test binary, built with `go test -c`
 *.test

Dockerfile

@@ -12,7 +12,7 @@ COPY go.sum go.sum
 RUN go mod download
 
 # Copy the go source
-COPY cmd/main.go cmd/main.go
+COPY cmd/ cmd/
 COPY api/ api/
 COPY internal/controller/ internal/controller/
 COPY internal/smolvm/ internal/smolvm/
@@ -22,13 +22,15 @@ COPY internal/smolvm/ internal/smolvm/
 # was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO
 # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore,
 # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform.
-RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/main.go
+RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/main.go && \
+    CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o runtime-agent cmd/runtime-agent/main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
 FROM gcr.io/distroless/static:nonroot
 WORKDIR /
 COPY --from=builder /workspace/manager .
+COPY --from=builder /workspace/runtime-agent .
 USER 65532:65532
 
 ENTRYPOINT ["/manager"]

Dockerfile.runtime

@@ -0,0 +1,35 @@
+FROM golang:1.25-bookworm AS agent-builder
+WORKDIR /workspace
+COPY go.mod go.mod
+COPY go.sum go.sum
+RUN go mod download
+COPY cmd/ cmd/
+COPY api/ api/
+COPY internal/smolvm/ internal/smolvm/
+RUN CGO_ENABLED=0 go build -a -o /runtime-agent cmd/runtime-agent/main.go
+
+FROM debian:bookworm-slim AS smolvm-release
+ARG TARGETARCH
+ARG SMOLVM_VERSION=0.8.1
+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl tar && rm -rf /var/lib/apt/lists/*
+WORKDIR /download
+RUN case "${TARGETARCH}" in \
+      amd64) SMOLVM_ARCH=x86_64 ;; \
+      arm64) SMOLVM_ARCH=aarch64 ;; \
+      *) echo "unsupported TARGETARCH=${TARGETARCH}" >&2; exit 1 ;; \
+    esac && \
+    curl -fsSL -o smolvm.tar.gz "https://github.com/smol-machines/smolvm/releases/download/v${SMOLVM_VERSION}/smolvm-${SMOLVM_VERSION}-linux-${SMOLVM_ARCH}.tar.gz" && \
+    mkdir -p /opt/smolvm && \
+    tar -xzf smolvm.tar.gz -C /opt/smolvm --strip-components=1
+
+FROM debian:bookworm-slim
+RUN apt-get update && apt-get install -y --no-install-recommends bash ca-certificates libepoxy0 libgbm1 libgcc-s1 libvirglrenderer1 && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /
+COPY --from=agent-builder /runtime-agent /runtime-agent
+COPY --from=smolvm-release /opt/smolvm/ /opt/smolvm/
+RUN chmod 0755 /runtime-agent /opt/smolvm/smolvm /opt/smolvm/smolvm-bin && \
+    ln -sf /opt/smolvm/smolvm /usr/local/bin/smolvm && \
+    mkdir -p /var/lib/smolvm /var/run/smolvm
+
+ENTRYPOINT ["/runtime-agent"]

Makefile

@@ -1,6 +1,8 @@
 
 # Image URL to use all building/pushing image targets
 IMG ?= controller:latest
+RUNTIME_IMG ?= smolvm-runtime:latest
+SMOLVM_VERSION ?= 0.8.1
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
 ENVTEST_K8S_VERSION = 1.29.0
 
@@ -64,11 +66,21 @@ vet: ## Run go vet against code.
 test: manifests generate fmt vet envtest ## Run tests.
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out
 
+.PHONY: test-report
+test-report: manifests generate fmt vet envtest gotestsum ## Run tests and write JUnit/coverage reports under reports/.
+	mkdir -p reports
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" $(GOTESTSUM) --format testname --junitfile reports/unit.xml -- -coverprofile reports/cover.out $$(go list ./... | grep -v /e2e)
+
 # Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors.
 .PHONY: test-e2e  # Run the e2e tests against a Kind k8s instance that is spun up.
 test-e2e:
 	go test ./test/e2e/ -v -ginkgo.v
 
+.PHONY: test-e2e-report
+test-e2e-report: gotestsum ## Run e2e tests and write a JUnit report under reports/.
+	mkdir -p reports
+	$(GOTESTSUM) --format testname --junitfile reports/e2e.xml -- ./test/e2e/ -v -ginkgo.v
+
 .PHONY: lint
 lint: golangci-lint ## Run golangci-lint linter & yamllint
 	$(GOLANGCI_LINT) run
@@ -82,6 +94,7 @@ lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes
 .PHONY: build
 build: manifests generate fmt vet ## Build manager binary.
 	go build -o bin/manager cmd/main.go
+	go build -o bin/runtime-agent cmd/runtime-agent/main.go
 
 .PHONY: run
 run: manifests generate fmt vet ## Run a controller from your host.
@@ -91,12 +104,17 @@ run: manifests generate fmt vet ## Run a controller from your host.
 # (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it.
 # More info: https://docs.docker.com/develop/develop-images/build_enhancements/
 .PHONY: docker-build
-docker-build: ## Build docker image with the manager.
+docker-build: ## Build docker image with the manager and runtime-agent.
 	$(CONTAINER_TOOL) build -t ${IMG} .
 
+.PHONY: docker-build-runtime
+docker-build-runtime: ## Build the smolvm runtime image, including the released smolvm binary.
+	$(CONTAINER_TOOL) build -f Dockerfile.runtime --build-arg SMOLVM_VERSION=$(SMOLVM_VERSION) -t ${RUNTIME_IMG} .
+
 .PHONY: docker-push
-docker-push: ## Push docker image with the manager.
+docker-push: ## Push docker images.
 	$(CONTAINER_TOOL) push ${IMG}
+	$(CONTAINER_TOOL) push ${RUNTIME_IMG}
 
 # PLATFORMS defines the target platforms for the manager image be built to provide support to multiple
 # architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to:
@@ -123,6 +141,7 @@ build-installer: manifests generate kustomize ## Generate a consolidated YAML wi
 	fi
 	echo "---" >> dist/install.yaml  # Add a document separator before appending
 	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+	cd config/runtime && $(KUSTOMIZE) edit set image smolvm-runtime=${RUNTIME_IMG}
 	$(KUSTOMIZE) build config/default >> dist/install.yaml
 
 ##@ Deployment
@@ -142,6 +161,7 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified
 .PHONY: deploy
 deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
 	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+	cd config/runtime && $(KUSTOMIZE) edit set image smolvm-runtime=${RUNTIME_IMG}
 	$(KUSTOMIZE) build config/default | $(KUBECTL) apply -f -
 
 .PHONY: undeploy
@@ -161,12 +181,14 @@ KUSTOMIZE ?= $(LOCALBIN)/kustomize-$(KUSTOMIZE_VERSION)
 CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen-$(CONTROLLER_TOOLS_VERSION)
 ENVTEST ?= $(LOCALBIN)/setup-envtest-$(ENVTEST_VERSION)
 GOLANGCI_LINT = $(LOCALBIN)/golangci-lint-$(GOLANGCI_LINT_VERSION)
+GOTESTSUM ?= $(LOCALBIN)/gotestsum-$(GOTESTSUM_VERSION)
 
 ## Tool Versions
 KUSTOMIZE_VERSION ?= v5.3.0
 CONTROLLER_TOOLS_VERSION ?= v0.19.0
 ENVTEST_VERSION ?= latest
 GOLANGCI_LINT_VERSION ?= v1.54.2
+GOTESTSUM_VERSION ?= v1.13.0
 
 .PHONY: kustomize
 kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary.
@@ -188,6 +210,11 @@ golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary.
 $(GOLANGCI_LINT): $(LOCALBIN)
 	$(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint,${GOLANGCI_LINT_VERSION})
 
+.PHONY: gotestsum
+gotestsum: $(GOTESTSUM) ## Download gotestsum locally if necessary.
+$(GOTESTSUM): $(LOCALBIN)
+	$(call go-install-tool,$(GOTESTSUM),gotest.tools/gotestsum,${GOTESTSUM_VERSION})
+
 # go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist
 # $1 - target path with name of binary (ideally with version)
 # $2 - package url which can be installed