Deploy controller as node-local DaemonSet

.github/workflows/ci.yml

@@ -7,39 +7,6 @@ on:
   workflow_dispatch:
 
 jobs:
-  kvm-check:
-    name: kvm check
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check runner KVM access
-        run: |
-          uname -a
-          ls -l /dev/kvm || true
-          sudo chmod 666 /dev/kvm
-          ls -l /dev/kvm
-          python3 - <<'PY'
-          import os
-          try:
-              fd = os.open("/dev/kvm", os.O_RDWR)
-              print("KVM open: OK")
-              os.close(fd)
-          except Exception as e:
-              print("KVM open: FAIL:", e)
-              raise SystemExit(1)
-          PY
-
-  unit:
-    name: unit
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-go@v5
-        with:
-          go-version-file: go.mod
-          cache: true
-      - name: Run one pure unit test
-        run: go test ./internal/controller -run TestBuildCreateRequestMapsSpec -count=1
-
   envtest:
     name: envtest controller
     runs-on: ubuntu-latest
@@ -145,10 +112,22 @@ jobs:
             curl -fsS http://127.0.0.1:8080/api/v1/machines || true
             exit 1
           fi
+          test "$(kubectl get smolvm runtime-smoke -o jsonpath='{.status.conditions[?(@.type=="RuntimeReady")].status}')" = "True"
+          test "$(kubectl get smolvm runtime-smoke -o jsonpath='{.status.conditions[?(@.type=="GuestReady")].status}')" = "Unknown"
 
           kubectl patch smolvm runtime-smoke --type=merge -p '{"spec":{"running":false}}'
           kubectl wait smolvm runtime-smoke \
             --for=jsonpath='{.status.phase}'=Stopped \
             --timeout=5m
 
-          kubectl delete smolvm runtime-smoke --wait=false
+          machine=$(kubectl get smolvm runtime-smoke -o jsonpath='{.status.machineName}')
+          kubectl delete smolvm runtime-smoke --wait=true --timeout=5m
+          for i in {1..30}; do
+            if ! curl -fsS http://127.0.0.1:8080/api/v1/machines | grep -q "$machine"; then
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "runtime machine $machine still exists after CR deletion"
+          curl -fsS http://127.0.0.1:8080/api/v1/machines || true
+          exit 1

README.md

@@ -89,3 +89,6 @@ Or over a Unix socket:
 ```sh
 SMOLVM_API_SOCKET=/var/run/smolvm/smolvm.sock make run
 ```
+
+The default Kubernetes DaemonSet manifest mounts `/var/run/smolvm` from each
+node and talks to `/var/run/smolvm/smolvm.sock`.

api/v1alpha1/smolvm_types.go

@@ -23,27 +23,33 @@ import (
 const (
 	SmolVMFinalizer = "smolvms.vm.smolvm.dev/finalizer"
 
-	ConditionReady      = "Ready"
-	ConditionReconciled = "Reconciled"
+	ConditionReady        = "Ready"
+	ConditionRuntimeReady = "RuntimeReady"
+	ConditionGuestReady   = "GuestReady"
+	ConditionReconciled   = "Reconciled"
 )
 
 // SmolVMSpec defines the desired state of a smolvm machine.
+// +kubebuilder:validation:XValidation:rule="(has(self.image) && size(self.image) > 0) != (has(self.from) && size(self.from) > 0)",message="exactly one of spec.image or spec.from is required"
+// +kubebuilder:validation:XValidation:rule="!has(oldSelf.nodeName) || (has(self.nodeName) && self.nodeName == oldSelf.nodeName)",message="spec.nodeName is immutable after binding"
 type SmolVMSpec struct {
 	// Running declares whether the machine should be running.
 	//+kubebuilder:default=true
-	Running bool `json:"running,omitempty"`
+	Running bool `json:"running"`
 
 	// NodeName pins the machine to a Kubernetes node. Machines are node-local and
 	// must not move implicitly after creation.
 	//+optional
 	NodeName string `json:"nodeName,omitempty"`
 
 	// Image is an OCI image to run inside the smolvm machine.
+	//+kubebuilder:validation:MinLength=1
 	//+optional
 	Image string `json:"image,omitempty"`
 
 	// From is a path to a .smolmachine sidecar artifact available to the node runtime.
 	// It is mutually exclusive with Image.
+	//+kubebuilder:validation:MinLength=1
 	//+optional
 	From string `json:"from,omitempty"`
 
@@ -93,10 +99,13 @@ type SmolVMNetwork struct {
 	Enabled bool `json:"enabled,omitempty"`
 
 	// AllowedCIDRs restricts egress to the listed CIDR ranges when supported by the runtime.
+	//+kubebuilder:validation:items:Format=cidr
 	//+optional
 	AllowedCIDRs []string `json:"allowedCIDRs,omitempty"`
 
 	// Ports maps host ports to guest ports on the selected node.
+	//+listType=map
+	//+listMapKey=hostPort
 	//+optional
 	Ports []SmolVMPort `json:"ports,omitempty"`
 }
@@ -128,6 +137,26 @@ type SmolVMStatus struct {
 	//+optional
 	MachineName string `json:"machineName,omitempty"`
 
+	// RuntimePID is the local process ID reported by the smolvm runtime when available.
+	//+optional
+	RuntimePID *int32 `json:"runtimePID,omitempty"`
+
+	// Network is the runtime-observed network enablement.
+	//+optional
+	Network bool `json:"network,omitempty"`
+
+	// Ports are the runtime-observed host-to-guest port mappings.
+	//+optional
+	Ports []SmolVMPort `json:"ports,omitempty"`
+
+	// StorageGiB is the runtime-observed storage disk size.
+	//+optional
+	StorageGiB *int64 `json:"storageGiB,omitempty"`
+
+	// OverlayGiB is the runtime-observed overlay disk size.
+	//+optional
+	OverlayGiB *int64 `json:"overlayGiB,omitempty"`
+
 	// ObservedGeneration is the metadata generation last reconciled by the controller.
 	//+optional
 	ObservedGeneration int64 `json:"observedGeneration,omitempty"`

config/crd/bases/vm.smolvm.dev_smolvms.yaml

@@ -56,9 +56,11 @@ spec:
                 description: |-
                   From is a path to a .smolmachine sidecar artifact available to the node runtime.
                   It is mutually exclusive with Image.
+                minLength: 1
                 type: string
               image:
                 description: Image is an OCI image to run inside the smolvm machine.
+                minLength: 1
                 type: string
               network:
                 description: Network configures outbound networking and host port
@@ -68,6 +70,7 @@ spec:
                     description: AllowedCIDRs restricts egress to the listed CIDR
                       ranges when supported by the runtime.
                     items:
+                      format: cidr
                       type: string
                     type: array
                   enabled:
@@ -96,6 +99,9 @@ spec:
                       - hostPort
                       type: object
                     type: array
+                    x-kubernetes-list-map-keys:
+                    - hostPort
+                    x-kubernetes-list-type: map
                 type: object
               nodeName:
                 description: |-
@@ -136,7 +142,16 @@ spec:
                     minimum: 1
                     type: integer
                 type: object
+            required:
+            - running
             type: object
+            x-kubernetes-validations:
+            - message: exactly one of spec.image or spec.from is required
+              rule: (has(self.image) && size(self.image) > 0) != (has(self.from) &&
+                size(self.from) > 0)
+            - message: spec.nodeName is immutable after binding
+              rule: '!has(oldSelf.nodeName) || (has(self.nodeName) && self.nodeName
+                == oldSelf.nodeName)'
           status:
             description: SmolVMStatus defines the observed state of a smolvm machine.
             properties:
@@ -201,6 +216,9 @@ spec:
                 description: MachineName is the stable smolvm runtime name owned by
                   this CR.
                 type: string
+              network:
+                description: Network is the runtime-observed network enablement.
+                type: boolean
               nodeName:
                 description: NodeName is the node currently owning this machine's
                   local state.
@@ -210,10 +228,45 @@ spec:
                   by the controller.
                 format: int64
                 type: integer
+              overlayGiB:
+                description: OverlayGiB is the runtime-observed overlay disk size.
+                format: int64
+                type: integer
               phase:
                 description: Phase is a compact machine phase such as Pending, Created,
                   Running, Stopped, Failed, or Unknown.
                 type: string
+              ports:
+                description: Ports are the runtime-observed host-to-guest port mappings.
+                items:
+                  description: SmolVMPort maps a node host port to a guest port.
+                  properties:
+                    guestPort:
+                      description: GuestPort is the TCP port inside the VM.
+                      format: int32
+                      maximum: 65535
+                      minimum: 1
+                      type: integer
+                    hostPort:
+                      description: HostPort is the TCP port opened on the node.
+                      format: int32
+                      maximum: 65535
+                      minimum: 1
+                      type: integer
+                  required:
+                  - guestPort
+                  - hostPort
+                  type: object
+                type: array
+              runtimePID:
+                description: RuntimePID is the local process ID reported by the smolvm
+                  runtime when available.
+                format: int32
+                type: integer
+              storageGiB:
+                description: StorageGiB is the runtime-observed storage disk size.
+                format: int64
+                type: integer
             type: object
         type: object
     served: true

config/default/manager_auth_proxy_patch.yaml

@@ -1,7 +1,7 @@
 # This patch inject a sidecar container which is a HTTP proxy for the
 # controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews.
 apiVersion: apps/v1
-kind: Deployment
+kind: DaemonSet
 metadata:
   name: controller-manager
   namespace: system
@@ -36,4 +36,3 @@ spec:
         args:
         - "--health-probe-bind-address=:8081"
         - "--metrics-bind-address=127.0.0.1:8080"
-        - "--leader-elect"

config/default/manager_config_patch.yaml

@@ -1,5 +1,5 @@
 apiVersion: apps/v1
-kind: Deployment
+kind: DaemonSet
 metadata:
   name: controller-manager
   namespace: system

config/manager/manager.yaml

@@ -12,13 +12,13 @@ metadata:
   name: system
 ---
 apiVersion: apps/v1
-kind: Deployment
+kind: DaemonSet
 metadata:
   name: controller-manager
   namespace: system
   labels:
     control-plane: controller-manager
-    app.kubernetes.io/name: deployment
+    app.kubernetes.io/name: daemonset
     app.kubernetes.io/instance: controller-manager
     app.kubernetes.io/component: manager
     app.kubernetes.io/created-by: operator
@@ -28,7 +28,6 @@ spec:
   selector:
     matchLabels:
       control-plane: controller-manager
-  replicas: 1
   template:
     metadata:
       annotations:
@@ -68,8 +67,13 @@ spec:
       containers:
       - command:
         - /manager
-        args:
-        - --leader-elect
+        env:
+        - name: SMOLVM_NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: SMOLVM_API_SOCKET
+          value: /var/run/smolvm/smolvm.sock
         image: controller:latest
         name: manager
         securityContext:
@@ -98,5 +102,14 @@ spec:
           requests:
             cpu: 10m
             memory: 64Mi
+        volumeMounts:
+        - name: smolvm-run
+          mountPath: /var/run/smolvm
+          readOnly: false
+      volumes:
+      - name: smolvm-run
+        hostPath:
+          path: /var/run/smolvm
+          type: DirectoryOrCreate
       serviceAccountName: controller-manager
       terminationGracePeriodSeconds: 10

config/rbac/kustomization.yaml

@@ -7,8 +7,6 @@ resources:
 - service_account.yaml
 - role.yaml
 - role_binding.yaml
-- leader_election_role.yaml
-- leader_election_role_binding.yaml
 # Comment the following 4 lines if you want to disable
 # the auth proxy (https://github.com/brancz/kube-rbac-proxy)
 # which protects your /metrics endpoint.

config/rbac/role.yaml

@@ -4,6 +4,13 @@ kind: ClusterRole
 metadata:
   name: manager-role
 rules:
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
 - apiGroups:
   - vm.smolvm.dev
   resources: