diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a49b63d..4e9b46e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,39 +7,6 @@ on: workflow_dispatch: jobs: - kvm-check: - name: kvm check - runs-on: ubuntu-latest - steps: - - name: Check runner KVM access - run: | - uname -a - ls -l /dev/kvm || true - sudo chmod 666 /dev/kvm - ls -l /dev/kvm - python3 - <<'PY' - import os - try: - fd = os.open("/dev/kvm", os.O_RDWR) - print("KVM open: OK") - os.close(fd) - except Exception as e: - print("KVM open: FAIL:", e) - raise SystemExit(1) - PY - - unit: - name: unit - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 - with: - go-version-file: go.mod - cache: true - - name: Run one pure unit test - run: go test ./internal/controller -run TestBuildCreateRequestMapsSpec -count=1 - envtest: name: envtest controller runs-on: ubuntu-latest @@ -145,10 +112,22 @@ jobs: curl -fsS http://127.0.0.1:8080/api/v1/machines || true exit 1 fi + test "$(kubectl get smolvm runtime-smoke -o jsonpath='{.status.conditions[?(@.type=="RuntimeReady")].status}')" = "True" + test "$(kubectl get smolvm runtime-smoke -o jsonpath='{.status.conditions[?(@.type=="GuestReady")].status}')" = "Unknown" kubectl patch smolvm runtime-smoke --type=merge -p '{"spec":{"running":false}}' kubectl wait smolvm runtime-smoke \ --for=jsonpath='{.status.phase}'=Stopped \ --timeout=5m - kubectl delete smolvm runtime-smoke --wait=false + machine=$(kubectl get smolvm runtime-smoke -o jsonpath='{.status.machineName}') + kubectl delete smolvm runtime-smoke --wait=true --timeout=5m + for i in {1..30}; do + if ! curl -fsS http://127.0.0.1:8080/api/v1/machines | grep -q "$machine"; then + exit 0 + fi + sleep 2 + done + echo "runtime machine $machine still exists after CR deletion" + curl -fsS http://127.0.0.1:8080/api/v1/machines || true + exit 1 diff --git a/README.md b/README.md index e7d2346..f36f364 100644 --- a/README.md +++ b/README.md @@ -89,3 +89,6 @@ Or over a Unix socket: ```sh SMOLVM_API_SOCKET=/var/run/smolvm/smolvm.sock make run ``` + +The default Kubernetes DaemonSet manifest mounts `/var/run/smolvm` from each +node and talks to `/var/run/smolvm/smolvm.sock`. diff --git a/api/v1alpha1/smolvm_types.go b/api/v1alpha1/smolvm_types.go index 9968469..5a90e24 100644 --- a/api/v1alpha1/smolvm_types.go +++ b/api/v1alpha1/smolvm_types.go @@ -23,15 +23,19 @@ import ( const ( SmolVMFinalizer = "smolvms.vm.smolvm.dev/finalizer" - ConditionReady = "Ready" - ConditionReconciled = "Reconciled" + ConditionReady = "Ready" + ConditionRuntimeReady = "RuntimeReady" + ConditionGuestReady = "GuestReady" + ConditionReconciled = "Reconciled" ) // SmolVMSpec defines the desired state of a smolvm machine. +// +kubebuilder:validation:XValidation:rule="(has(self.image) && size(self.image) > 0) != (has(self.from) && size(self.from) > 0)",message="exactly one of spec.image or spec.from is required" +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.nodeName) || (has(self.nodeName) && self.nodeName == oldSelf.nodeName)",message="spec.nodeName is immutable after binding" type SmolVMSpec struct { // Running declares whether the machine should be running. //+kubebuilder:default=true - Running bool `json:"running,omitempty"` + Running bool `json:"running"` // NodeName pins the machine to a Kubernetes node. Machines are node-local and // must not move implicitly after creation. @@ -39,11 +43,13 @@ type SmolVMSpec struct { NodeName string `json:"nodeName,omitempty"` // Image is an OCI image to run inside the smolvm machine. + //+kubebuilder:validation:MinLength=1 //+optional Image string `json:"image,omitempty"` // From is a path to a .smolmachine sidecar artifact available to the node runtime. // It is mutually exclusive with Image. + //+kubebuilder:validation:MinLength=1 //+optional From string `json:"from,omitempty"` @@ -93,10 +99,13 @@ type SmolVMNetwork struct { Enabled bool `json:"enabled,omitempty"` // AllowedCIDRs restricts egress to the listed CIDR ranges when supported by the runtime. + //+kubebuilder:validation:items:Format=cidr //+optional AllowedCIDRs []string `json:"allowedCIDRs,omitempty"` // Ports maps host ports to guest ports on the selected node. + //+listType=map + //+listMapKey=hostPort //+optional Ports []SmolVMPort `json:"ports,omitempty"` } @@ -128,6 +137,26 @@ type SmolVMStatus struct { //+optional MachineName string `json:"machineName,omitempty"` + // RuntimePID is the local process ID reported by the smolvm runtime when available. + //+optional + RuntimePID *int32 `json:"runtimePID,omitempty"` + + // Network is the runtime-observed network enablement. + //+optional + Network bool `json:"network,omitempty"` + + // Ports are the runtime-observed host-to-guest port mappings. + //+optional + Ports []SmolVMPort `json:"ports,omitempty"` + + // StorageGiB is the runtime-observed storage disk size. + //+optional + StorageGiB *int64 `json:"storageGiB,omitempty"` + + // OverlayGiB is the runtime-observed overlay disk size. + //+optional + OverlayGiB *int64 `json:"overlayGiB,omitempty"` + // ObservedGeneration is the metadata generation last reconciled by the controller. //+optional ObservedGeneration int64 `json:"observedGeneration,omitempty"` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index b059465..167a21d 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -160,6 +160,26 @@ func (in *SmolVMSpec) DeepCopy() *SmolVMSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SmolVMStatus) DeepCopyInto(out *SmolVMStatus) { *out = *in + if in.RuntimePID != nil { + in, out := &in.RuntimePID, &out.RuntimePID + *out = new(int32) + **out = **in + } + if in.Ports != nil { + in, out := &in.Ports, &out.Ports + *out = make([]SmolVMPort, len(*in)) + copy(*out, *in) + } + if in.StorageGiB != nil { + in, out := &in.StorageGiB, &out.StorageGiB + *out = new(int64) + **out = **in + } + if in.OverlayGiB != nil { + in, out := &in.OverlayGiB, &out.OverlayGiB + *out = new(int64) + **out = **in + } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make([]v1.Condition, len(*in)) diff --git a/config/crd/bases/vm.smolvm.dev_smolvms.yaml b/config/crd/bases/vm.smolvm.dev_smolvms.yaml index 68ad23a..d4b96bb 100644 --- a/config/crd/bases/vm.smolvm.dev_smolvms.yaml +++ b/config/crd/bases/vm.smolvm.dev_smolvms.yaml @@ -56,9 +56,11 @@ spec: description: |- From is a path to a .smolmachine sidecar artifact available to the node runtime. It is mutually exclusive with Image. + minLength: 1 type: string image: description: Image is an OCI image to run inside the smolvm machine. + minLength: 1 type: string network: description: Network configures outbound networking and host port @@ -68,6 +70,7 @@ spec: description: AllowedCIDRs restricts egress to the listed CIDR ranges when supported by the runtime. items: + format: cidr type: string type: array enabled: @@ -96,6 +99,9 @@ spec: - hostPort type: object type: array + x-kubernetes-list-map-keys: + - hostPort + x-kubernetes-list-type: map type: object nodeName: description: |- @@ -136,7 +142,16 @@ spec: minimum: 1 type: integer type: object + required: + - running type: object + x-kubernetes-validations: + - message: exactly one of spec.image or spec.from is required + rule: (has(self.image) && size(self.image) > 0) != (has(self.from) && + size(self.from) > 0) + - message: spec.nodeName is immutable after binding + rule: '!has(oldSelf.nodeName) || (has(self.nodeName) && self.nodeName + == oldSelf.nodeName)' status: description: SmolVMStatus defines the observed state of a smolvm machine. properties: @@ -201,6 +216,9 @@ spec: description: MachineName is the stable smolvm runtime name owned by this CR. type: string + network: + description: Network is the runtime-observed network enablement. + type: boolean nodeName: description: NodeName is the node currently owning this machine's local state. @@ -210,10 +228,45 @@ spec: by the controller. format: int64 type: integer + overlayGiB: + description: OverlayGiB is the runtime-observed overlay disk size. + format: int64 + type: integer phase: description: Phase is a compact machine phase such as Pending, Created, Running, Stopped, Failed, or Unknown. type: string + ports: + description: Ports are the runtime-observed host-to-guest port mappings. + items: + description: SmolVMPort maps a node host port to a guest port. + properties: + guestPort: + description: GuestPort is the TCP port inside the VM. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + hostPort: + description: HostPort is the TCP port opened on the node. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - guestPort + - hostPort + type: object + type: array + runtimePID: + description: RuntimePID is the local process ID reported by the smolvm + runtime when available. + format: int32 + type: integer + storageGiB: + description: StorageGiB is the runtime-observed storage disk size. + format: int64 + type: integer type: object type: object served: true diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml index 70c3437..90649d4 100644 --- a/config/default/manager_auth_proxy_patch.yaml +++ b/config/default/manager_auth_proxy_patch.yaml @@ -1,7 +1,7 @@ # This patch inject a sidecar container which is a HTTP proxy for the # controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. apiVersion: apps/v1 -kind: Deployment +kind: DaemonSet metadata: name: controller-manager namespace: system @@ -36,4 +36,3 @@ spec: args: - "--health-probe-bind-address=:8081" - "--metrics-bind-address=127.0.0.1:8080" - - "--leader-elect" diff --git a/config/default/manager_config_patch.yaml b/config/default/manager_config_patch.yaml index f6f5891..1c9f0de 100644 --- a/config/default/manager_config_patch.yaml +++ b/config/default/manager_config_patch.yaml @@ -1,5 +1,5 @@ apiVersion: apps/v1 -kind: Deployment +kind: DaemonSet metadata: name: controller-manager namespace: system diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 67867c0..14a461d 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -12,13 +12,13 @@ metadata: name: system --- apiVersion: apps/v1 -kind: Deployment +kind: DaemonSet metadata: name: controller-manager namespace: system labels: control-plane: controller-manager - app.kubernetes.io/name: deployment + app.kubernetes.io/name: daemonset app.kubernetes.io/instance: controller-manager app.kubernetes.io/component: manager app.kubernetes.io/created-by: operator @@ -28,7 +28,6 @@ spec: selector: matchLabels: control-plane: controller-manager - replicas: 1 template: metadata: annotations: @@ -68,8 +67,13 @@ spec: containers: - command: - /manager - args: - - --leader-elect + env: + - name: SMOLVM_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: SMOLVM_API_SOCKET + value: /var/run/smolvm/smolvm.sock image: controller:latest name: manager securityContext: @@ -98,5 +102,14 @@ spec: requests: cpu: 10m memory: 64Mi + volumeMounts: + - name: smolvm-run + mountPath: /var/run/smolvm + readOnly: false + volumes: + - name: smolvm-run + hostPath: + path: /var/run/smolvm + type: DirectoryOrCreate serviceAccountName: controller-manager terminationGracePeriodSeconds: 10 diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 731832a..4e42541 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -7,8 +7,6 @@ resources: - service_account.yaml - role.yaml - role_binding.yaml -- leader_election_role.yaml -- leader_election_role_binding.yaml # Comment the following 4 lines if you want to disable # the auth proxy (https://github.com/brancz/kube-rbac-proxy) # which protects your /metrics endpoint. diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index f0df6a0..14d5aac 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -4,6 +4,13 @@ kind: ClusterRole metadata: name: manager-role rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch - apiGroups: - vm.smolvm.dev resources: diff --git a/docs/e2e-test-feasibility.md b/docs/e2e-test-feasibility.md new file mode 100644 index 0000000..a8752c4 --- /dev/null +++ b/docs/e2e-test-feasibility.md @@ -0,0 +1,53 @@ +# E2E test feasibility + +| # | Test | Feasibility against smolvm | Operator readiness | Notes | +| --- | --- | --- | --- | --- | +| 1 | CRD install / API discovery | Feasible | Ready | Kubernetes-only test. | +| 2 | Manager deployment health | Feasible | Ready as DaemonSet | Use DaemonSet rollout status, not Deployment availability. | +| 3 | Invalid spec handling | Feasible | Ready | Controller marks invalid specs as `Failed`; admission rejection is not implemented. | +| 4 | Runtime unavailable behavior | Feasible | Ready | Kubernetes-only negative test with no `smolvm serve`. | +| 5 | Create + start lifecycle | Feasible | Ready | smolvm supports create and start; CI must provide `/dev/kvm` and libkrun runtime libraries. | +| 6 | Create stopped lifecycle | Feasible | Ready | smolvm create returns `created`; controller reports non-running as not Ready but reconciled. | +| 7 | Start after patch | Feasible | Ready | Patch `spec.running` from `false` to `true`; controller starts the existing machine. | +| 8 | Stop after patch | Feasible | Ready | smolvm supports stop; status should become `Stopped`. | +| 9 | Delete finalizer cleanup | Feasible | Ready | smolvm supports delete and stops the VM first if needed. | +| 10 | Delete when runtime machine already gone | Feasible | Ready | smolvm returns 404; controller already ignores not-found on delete. | +| 11 | Storage shrink rejection | Feasible | Ready | Both smolvm and controller reject shrink; controller should report `InvalidStorageResize`. | +| 12 | Storage expansion while running | Feasible | Ready | smolvm rejects resize while running; controller prevents the call and reports stop-required. | +| 13 | Storage expansion while stopped | Feasible | Mostly ready | smolvm supports stopped resize; test should use small sizes because disk expansion costs CI time and space. | +| 14 | Node filtering | Feasible | Ready | smolvm has no node field; this is operator-side filtering via `SMOLVM_NODE_NAME`. | +| 15 | Node-local missing assignment | Feasible | Ready | Operator-side behavior when `SMOLVM_NODE_NAME` is set and `spec.nodeName` is empty. | +| 16 | Status observedGeneration | Feasible | Ready | Kubernetes-only assertion after create/patch reconciliation. | +| 17 | Reconciliation idempotency | Feasible | Ready | smolvm create is not used repeatedly once machine exists; assert one stable machine name. | +| 18 | Controller restart recovery | Feasible | Ready | smolvm persists machine records and API can observe existing machines after controller restart. | +| 19 | Spec drift behavior | Partially feasible | Not mature | smolvm create-time fields such as image, CPU, memory, network, ports are not updated by current operator. Mature behavior needs immutability/admission or explicit drift status before this can be a passing test. | +| 20 | Host port / networking contract | Partially feasible | Not mature | smolvm supports `network`, `ports`, and `allowedCidrs` at create/start. The operator maps these fields but does not reserve host ports or detect cluster-wide conflicts. Test positive mapping first; conflict tests need new validation/status behavior. | + +## Recommended mature e2e suite + +### Always-on CI + +- CRD install / discovery. +- DaemonSet rollout. +- Invalid spec status. +- Runtime-unavailable status. +- Node filtering and missing assignment. +- Idempotency using a fake or unavailable runtime. + +### KVM runtime CI + +- Create running machine. +- Create stopped machine. +- Patch start. +- Patch stop. +- Delete finalizer cleanup. +- Delete after out-of-band runtime deletion. +- Storage shrink rejection. +- Resize requires stopped machine. +- Stopped storage expansion. +- Controller restart recovery. + +### Requires operator changes first + +- Spec drift must be defined as either immutable, mutable, or explicitly unreconciled. +- Host port conflicts must be validated or surfaced deterministically. diff --git a/internal/controller/smolvm_controller.go b/internal/controller/smolvm_controller.go index 2798865..e2e1caf 100644 --- a/internal/controller/smolvm_controller.go +++ b/internal/controller/smolvm_controller.go @@ -21,12 +21,16 @@ import ( "crypto/sha256" "encoding/hex" "fmt" + "net" "os" + "sort" "time" + apiequality "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -44,6 +48,7 @@ type SmolVMReconciler struct { Scheme *runtime.Scheme RuntimeFactory func() SmolVMRuntime + Recorder record.EventRecorder } type SmolVMRuntime interface { @@ -65,6 +70,7 @@ func (r *SmolVMReconciler) runtimeClient() SmolVMRuntime { //+kubebuilder:rbac:groups=vm.smolvm.dev,resources=smolvms,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=vm.smolvm.dev,resources=smolvms/status,verbs=get;update;patch //+kubebuilder:rbac:groups=vm.smolvm.dev,resources=smolvms/finalizers,verbs=update +//+kubebuilder:rbac:groups="",resources=events,verbs=create;patch func (r *SmolVMReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { logger := log.FromContext(ctx) @@ -76,8 +82,9 @@ func (r *SmolVMReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr runtimeNode := os.Getenv("SMOLVM_NODE_NAME") desiredNode := vm.Spec.NodeName - if desiredNode == "" { - desiredNode = vm.Status.NodeName + ownerNode := vm.Status.NodeName + if ownerNode == "" { + ownerNode = desiredNode } machineName := vm.Status.MachineName @@ -85,7 +92,7 @@ func (r *SmolVMReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr machineName = stableMachineName(&vm) } - if runtimeNode != "" && desiredNode == "" { + if runtimeNode != "" && ownerNode == "" { return r.updateStatus(ctx, &vm, statusInput{ Phase: "Pending", NodeName: "", @@ -99,14 +106,14 @@ func (r *SmolVMReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr }) } - if desiredNode != "" && runtimeNode != "" && desiredNode != runtimeNode { + if ownerNode != "" && runtimeNode != "" && ownerNode != runtimeNode { return ctrl.Result{}, nil } api := r.runtimeClient() if !vm.ObjectMeta.DeletionTimestamp.IsZero() { - return r.reconcileDelete(ctx, &vm, api, machineName) + return r.reconcileDelete(ctx, &vm, api, machineName, ownerNode) } if !controllerutil.ContainsFinalizer(&vm, vmv1alpha1.SmolVMFinalizer) { @@ -116,9 +123,10 @@ func (r *SmolVMReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr if err := validateSpec(&vm); err != nil { logger.Info("invalid SmolVM spec", "error", err.Error()) + r.event(&vm, "Warning", "InvalidSpec", err.Error()) return r.updateStatus(ctx, &vm, statusInput{ Phase: "Failed", - NodeName: desiredNode, + NodeName: ownerNode, MachineName: machineName, Ready: metav1.ConditionFalse, ReadyReason: "InvalidSpec", @@ -133,14 +141,16 @@ func (r *SmolVMReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr if smolvmapi.IsNotFound(err) { createReq := buildCreateRequest(&vm, machineName) logger.Info("creating smolvm machine", "machine", machineName) + r.event(&vm, "Normal", "Creating", fmt.Sprintf("creating smolvm machine %s", machineName)) machine, err = api.CreateMachine(ctx, createReq) if err != nil { - return r.runtimeUnavailable(ctx, &vm, desiredNode, machineName, err) + return r.runtimeUnavailable(ctx, &vm, ownerNode, machineName, err) } _, statusErr := r.updateStatus(ctx, &vm, statusInput{ Phase: phaseFromMachine(machine), - NodeName: desiredNode, + NodeName: ownerNode, MachineName: machineName, + Machine: machine, Ready: metav1.ConditionFalse, ReadyReason: "Created", ReadyMsg: "smolvm machine has been created; waiting before start", @@ -154,14 +164,31 @@ func (r *SmolVMReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr return ctrl.Result{RequeueAfter: requeueAfter}, nil } if err != nil { - return r.runtimeUnavailable(ctx, &vm, desiredNode, machineName, err) + return r.runtimeUnavailable(ctx, &vm, ownerNode, machineName, err) + } + + if immutableErr := validateImmutableRuntimeFields(&vm, machine); immutableErr != nil { + r.event(&vm, "Warning", "UnsupportedUpdate", immutableErr.Error()) + return r.updateStatus(ctx, &vm, statusInput{ + Phase: phaseFromMachine(machine), + NodeName: ownerNode, + MachineName: machineName, + Machine: machine, + Ready: readyFromMachine(machine), + ReadyReason: "Observed", + ReadyMsg: fmt.Sprintf("smolvm machine is %s", machine.State), + Recon: metav1.ConditionFalse, + ReconReason: "UnsupportedUpdate", + ReconMsg: immutableErr.Error(), + }) } if invalidResize := validateStorageDoesNotShrink(&vm, machine); invalidResize != nil { return r.updateStatus(ctx, &vm, statusInput{ Phase: phaseFromMachine(machine), - NodeName: desiredNode, + NodeName: ownerNode, MachineName: machineName, + Machine: machine, Ready: readyFromMachine(machine), ReadyReason: "Observed", ReadyMsg: fmt.Sprintf("smolvm machine is %s", machine.State), @@ -175,8 +202,9 @@ func (r *SmolVMReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr if machine.State == "running" { return r.updateStatus(ctx, &vm, statusInput{ Phase: phaseFromMachine(machine), - NodeName: desiredNode, + NodeName: ownerNode, MachineName: machineName, + Machine: machine, Ready: readyFromMachine(machine), ReadyReason: "ResizePending", ReadyMsg: "storage expansion requires the machine to be stopped", @@ -187,31 +215,35 @@ func (r *SmolVMReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr } resize := buildResizeRequest(&vm) logger.Info("resizing smolvm machine", "machine", machineName) + r.event(&vm, "Normal", "Resizing", fmt.Sprintf("resizing smolvm machine %s", machineName)) if err := api.ResizeMachine(ctx, machineName, resize); err != nil { - return r.runtimeUnavailable(ctx, &vm, desiredNode, machineName, err) + return r.runtimeUnavailable(ctx, &vm, ownerNode, machineName, err) } return ctrl.Result{RequeueAfter: requeueAfter}, nil } if vm.Spec.Running && machine.State != "running" { logger.Info("starting smolvm machine", "machine", machineName, "state", machine.State) + r.event(&vm, "Normal", "Starting", fmt.Sprintf("starting smolvm machine %s", machineName)) if err := api.EnsureMachineRunning(ctx, machineName); err != nil { - return r.runtimeUnavailable(ctx, &vm, desiredNode, machineName, err) + return r.runtimeUnavailable(ctx, &vm, ownerNode, machineName, err) } return ctrl.Result{RequeueAfter: requeueAfter}, nil } if !vm.Spec.Running && machine.State == "running" { logger.Info("stopping smolvm machine", "machine", machineName) + r.event(&vm, "Normal", "Stopping", fmt.Sprintf("stopping smolvm machine %s", machineName)) if err := api.StopMachine(ctx, machineName); err != nil { - return r.runtimeUnavailable(ctx, &vm, desiredNode, machineName, err) + return r.runtimeUnavailable(ctx, &vm, ownerNode, machineName, err) } return ctrl.Result{RequeueAfter: requeueAfter}, nil } result, err := r.updateStatus(ctx, &vm, statusInput{ Phase: phaseFromMachine(machine), - NodeName: desiredNode, + NodeName: ownerNode, MachineName: machineName, + Machine: machine, Ready: readyFromMachine(machine), ReadyReason: "Observed", ReadyMsg: fmt.Sprintf("smolvm machine is %s", machine.State), @@ -225,10 +257,11 @@ func (r *SmolVMReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr return ctrl.Result{RequeueAfter: requeueAfter}, nil } -func (r *SmolVMReconciler) reconcileDelete(ctx context.Context, vm *vmv1alpha1.SmolVM, api SmolVMRuntime, machineName string) (ctrl.Result, error) { +func (r *SmolVMReconciler) reconcileDelete(ctx context.Context, vm *vmv1alpha1.SmolVM, api SmolVMRuntime, machineName, ownerNode string) (ctrl.Result, error) { if machineName != "" { + r.event(vm, "Normal", "Deleting", fmt.Sprintf("deleting smolvm machine %s", machineName)) if err := api.DeleteMachine(ctx, machineName); err != nil && !smolvmapi.IsNotFound(err) { - return r.runtimeUnavailable(ctx, vm, vm.Status.NodeName, machineName, err) + return r.runtimeUnavailable(ctx, vm, ownerNode, machineName, err) } } controllerutil.RemoveFinalizer(vm, vmv1alpha1.SmolVMFinalizer) @@ -236,6 +269,7 @@ func (r *SmolVMReconciler) reconcileDelete(ctx context.Context, vm *vmv1alpha1.S } func (r *SmolVMReconciler) runtimeUnavailable(ctx context.Context, vm *vmv1alpha1.SmolVM, nodeName, machineName string, err error) (ctrl.Result, error) { + r.event(vm, "Warning", "RuntimeUnavailable", err.Error()) _, statusErr := r.updateStatus(ctx, vm, statusInput{ Phase: "Unknown", NodeName: nodeName, @@ -257,6 +291,7 @@ type statusInput struct { Phase string NodeName string MachineName string + Machine *smolvmapi.MachineInfo Ready metav1.ConditionStatus ReadyReason string ReadyMsg string @@ -270,12 +305,25 @@ func (r *SmolVMReconciler) updateStatus(ctx context.Context, vm *vmv1alpha1.Smol if err := r.Get(ctx, types.NamespacedName{Name: vm.Name, Namespace: vm.Namespace}, latest); err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } + previous := latest.Status.DeepCopy() latest.Status.Phase = in.Phase latest.Status.NodeName = in.NodeName latest.Status.MachineName = in.MachineName + if in.Machine != nil { + latest.Status.RuntimePID = in.Machine.PID + latest.Status.Network = in.Machine.Network + latest.Status.Ports = portsFromRuntime(in.Machine.Ports) + latest.Status.StorageGiB = in.Machine.StorageGB + latest.Status.OverlayGiB = in.Machine.OverlayGB + } latest.Status.ObservedGeneration = latest.Generation setCondition(&latest.Status.Conditions, vmv1alpha1.ConditionReady, in.Ready, in.ReadyReason, in.ReadyMsg, latest.Generation) + setCondition(&latest.Status.Conditions, vmv1alpha1.ConditionRuntimeReady, in.Ready, in.ReadyReason, in.ReadyMsg, latest.Generation) + setCondition(&latest.Status.Conditions, vmv1alpha1.ConditionGuestReady, metav1.ConditionUnknown, "GuestReadinessUnavailable", "smolvm guest readiness is not reported by the runtime API", latest.Generation) setCondition(&latest.Status.Conditions, vmv1alpha1.ConditionReconciled, in.Recon, in.ReconReason, in.ReconMsg, latest.Generation) + if apiequality.Semantic.DeepEqual(previous, &latest.Status) { + return ctrl.Result{}, nil + } return ctrl.Result{}, r.Status().Update(ctx, latest) } @@ -303,12 +351,27 @@ func setCondition(conditions *[]metav1.Condition, typ string, status metav1.Cond } func validateSpec(vm *vmv1alpha1.SmolVM) error { + if vm.Status.NodeName != "" && vm.Spec.NodeName != "" && vm.Spec.NodeName != vm.Status.NodeName { + return fmt.Errorf("spec.nodeName is immutable after binding; machine is owned by node %q", vm.Status.NodeName) + } if vm.Spec.Image != "" && vm.Spec.From != "" { return fmt.Errorf("spec.image and spec.from are mutually exclusive") } if vm.Spec.Image == "" && vm.Spec.From == "" { return fmt.Errorf("one of spec.image or spec.from is required") } + seenPorts := map[int32]struct{}{} + for _, port := range vm.Spec.Network.Ports { + if _, ok := seenPorts[port.HostPort]; ok { + return fmt.Errorf("duplicate hostPort %d", port.HostPort) + } + seenPorts[port.HostPort] = struct{}{} + } + for _, cidr := range vm.Spec.Network.AllowedCIDRs { + if _, _, err := net.ParseCIDR(cidr); err != nil { + return fmt.Errorf("invalid allowedCIDR %q: %w", cidr, err) + } + } return nil } @@ -345,6 +408,52 @@ func buildResizeRequest(vm *vmv1alpha1.SmolVM) smolvmapi.ResizeRequest { return req } +func validateImmutableRuntimeFields(vm *vmv1alpha1.SmolVM, machine *smolvmapi.MachineInfo) error { + if vm.Spec.Resources.CPUs > 0 && machine.CPUs > 0 && vm.Spec.Resources.CPUs != machine.CPUs { + return fmt.Errorf("cpus is immutable after creation; runtime has %d and spec requests %d", machine.CPUs, vm.Spec.Resources.CPUs) + } + if vm.Spec.Resources.MemoryMiB > 0 && machine.MemoryMiB > 0 && vm.Spec.Resources.MemoryMiB != machine.MemoryMiB { + return fmt.Errorf("memoryMiB is immutable after creation; runtime has %d and spec requests %d", machine.MemoryMiB, vm.Spec.Resources.MemoryMiB) + } + if !samePorts(vm.Spec.Network.Ports, machine.Ports) { + return fmt.Errorf("network port mappings are immutable after creation") + } + return nil +} + +func samePorts(spec []vmv1alpha1.SmolVMPort, runtime []smolvmapi.PortSpec) bool { + if len(spec) != len(runtime) { + return false + } + specPorts := make([]string, 0, len(spec)) + for _, p := range spec { + specPorts = append(specPorts, fmt.Sprintf("%d:%d", p.HostPort, p.GuestPort)) + } + runtimePorts := make([]string, 0, len(runtime)) + for _, p := range runtime { + runtimePorts = append(runtimePorts, fmt.Sprintf("%d:%d", p.Host, p.Guest)) + } + sort.Strings(specPorts) + sort.Strings(runtimePorts) + for i := range specPorts { + if specPorts[i] != runtimePorts[i] { + return false + } + } + return true +} + +func portsFromRuntime(ports []smolvmapi.PortSpec) []vmv1alpha1.SmolVMPort { + if len(ports) == 0 { + return nil + } + out := make([]vmv1alpha1.SmolVMPort, 0, len(ports)) + for _, port := range ports { + out = append(out, vmv1alpha1.SmolVMPort{HostPort: port.Host, GuestPort: port.Guest}) + } + return out +} + func validateStorageDoesNotShrink(vm *vmv1alpha1.SmolVM, machine *smolvmapi.MachineInfo) error { if vm.Spec.Storage.StorageGiB > 0 && machine.StorageGB != nil && vm.Spec.Storage.StorageGiB < *machine.StorageGB { return fmt.Errorf("storageGiB cannot shrink from %d to %d", *machine.StorageGB, vm.Spec.Storage.StorageGiB) @@ -386,8 +495,17 @@ func stableMachineName(vm *vmv1alpha1.SmolVM) string { return fmt.Sprintf("k8s-%s-%s-%s", vm.Namespace, vm.Name, uid) } +func (r *SmolVMReconciler) event(vm *vmv1alpha1.SmolVM, eventType, reason, message string) { + if r.Recorder != nil { + r.Recorder.Event(vm, eventType, reason, message) + } +} + // SetupWithManager sets up the controller with the Manager. func (r *SmolVMReconciler) SetupWithManager(mgr ctrl.Manager) error { + if r.Recorder == nil { + r.Recorder = mgr.GetEventRecorderFor("smolvm-controller") + } return ctrl.NewControllerManagedBy(mgr). For(&vmv1alpha1.SmolVM{}). Complete(r) diff --git a/internal/controller/smolvm_controller_test.go b/internal/controller/smolvm_controller_test.go index 3d65777..aeff50c 100644 --- a/internal/controller/smolvm_controller_test.go +++ b/internal/controller/smolvm_controller_test.go @@ -2,19 +2,81 @@ package controller import ( "context" + "fmt" "net/http" + "os" "testing" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/reconcile" vmv1alpha1 "github.com/manuschillerdev/smolvm-operator/api/v1alpha1" smolvmapi "github.com/manuschillerdev/smolvm-operator/internal/smolvm" ) +func TestValidateSpecRejectsInvalidNetworkInputs(t *testing.T) { + vm := &vmv1alpha1.SmolVM{ + Spec: vmv1alpha1.SmolVMSpec{ + Image: "alpine:latest", + Network: vmv1alpha1.SmolVMNetwork{ + AllowedCIDRs: []string{"not-a-cidr"}, + }, + }, + } + if err := validateSpec(vm); err == nil { + t.Fatal("expected invalid CIDR to be rejected") + } + + vm.Spec.Network.AllowedCIDRs = nil + vm.Spec.Network.Ports = []vmv1alpha1.SmolVMPort{ + {HostPort: 8080, GuestPort: 80}, + {HostPort: 8080, GuestPort: 8080}, + } + if err := validateSpec(vm); err == nil { + t.Fatal("expected duplicate hostPort to be rejected") + } +} + +func TestValidateSpecRejectsNodeMoveAfterBinding(t *testing.T) { + vm := &vmv1alpha1.SmolVM{ + Spec: vmv1alpha1.SmolVMSpec{NodeName: "node-b", Image: "alpine:latest"}, + Status: vmv1alpha1.SmolVMStatus{NodeName: "node-a"}, + } + if err := validateSpec(vm); err == nil { + t.Fatal("expected nodeName mutation to be rejected") + } +} + +func TestValidateImmutableRuntimeFieldsRejectsUnsupportedDrift(t *testing.T) { + vm := &vmv1alpha1.SmolVM{ + Spec: vmv1alpha1.SmolVMSpec{ + Image: "alpine:latest", + Resources: vmv1alpha1.SmolVMResources{CPUs: 2, MemoryMiB: 256}, + Network: vmv1alpha1.SmolVMNetwork{Ports: []vmv1alpha1.SmolVMPort{{HostPort: 8080, GuestPort: 80}}}, + }, + } + machine := &smolvmapi.MachineInfo{ + CPUs: 1, + MemoryMiB: 256, + Ports: []smolvmapi.PortSpec{{Host: 8080, Guest: 80}}, + } + if err := validateImmutableRuntimeFields(vm, machine); err == nil { + t.Fatal("expected CPU drift to be rejected") + } + + machine.CPUs = 2 + machine.Ports = []smolvmapi.PortSpec{{Host: 9090, Guest: 80}} + if err := validateImmutableRuntimeFields(vm, machine); err == nil { + t.Fatal("expected port drift to be rejected") + } +} + func TestBuildCreateRequestMapsSpec(t *testing.T) { vm := &vmv1alpha1.SmolVM{ ObjectMeta: metav1.ObjectMeta{Name: "alpine", Namespace: "default"}, @@ -46,27 +108,11 @@ var _ = Describe("SmolVM Controller", func() { It("creates a missing runtime machine and records status", func() { ctx := context.Background() name := types.NamespacedName{Name: "runtime-create", Namespace: "default"} - resource := &vmv1alpha1.SmolVM{ - ObjectMeta: metav1.ObjectMeta{Name: name.Name, Namespace: name.Namespace}, - Spec: vmv1alpha1.SmolVMSpec{ - Running: true, - Image: "alpine:latest", - Resources: vmv1alpha1.SmolVMResources{ - CPUs: 1, - MemoryMiB: 128, - }, - }, - } + resource := testSmolVM(name, "") Expect(k8sClient.Create(ctx, resource)).To(Succeed()) runtime := &fakeRuntime{} - reconciler := &SmolVMReconciler{ - Client: k8sClient, - Scheme: k8sClient.Scheme(), - RuntimeFactory: func() SmolVMRuntime { - return runtime - }, - } + reconciler := testReconciler(runtime) _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: name}) Expect(err).NotTo(HaveOccurred()) @@ -80,29 +126,457 @@ var _ = Describe("SmolVM Controller", func() { Expect(latest.Status.MachineName).NotTo(BeEmpty()) Expect(latest.Status.Conditions).NotTo(BeEmpty()) }) + + It("adopts an existing runtime machine after restart", func() { + ctx := context.Background() + name := types.NamespacedName{Name: "runtime-adopt", Namespace: "default"} + machineName := "k8s-default-runtime-adopt-test" + Expect(createSmolVMWithStatus(ctx, name, "node-a", machineName)).To(Succeed()) + + runtime := &fakeRuntime{machine: &smolvmapi.MachineInfo{Name: machineName, State: "running", CPUs: 1, MemoryMiB: 128}} + _, err := testReconciler(runtime).Reconcile(ctx, reconcile.Request{NamespacedName: name}) + Expect(err).NotTo(HaveOccurred()) + + Expect(runtime.created).To(Equal(0)) + latest := &vmv1alpha1.SmolVM{} + Expect(k8sClient.Get(ctx, name, latest)).To(Succeed()) + Expect(latest.Status.Phase).To(Equal("Running")) + Expect(conditionReason(latest, vmv1alpha1.ConditionReconciled)).To(Equal("Reconciled")) + }) + + It("marks unsupported runtime drift without mutating the machine", func() { + ctx := context.Background() + name := types.NamespacedName{Name: "runtime-drift", Namespace: "default"} + machineName := "k8s-default-runtime-drift-test" + Expect(createSmolVMWithStatus(ctx, name, "node-a", machineName)).To(Succeed()) + + runtime := &fakeRuntime{machine: &smolvmapi.MachineInfo{Name: machineName, State: "running", CPUs: 2, MemoryMiB: 128}} + _, err := testReconciler(runtime).Reconcile(ctx, reconcile.Request{NamespacedName: name}) + Expect(err).NotTo(HaveOccurred()) + + latest := &vmv1alpha1.SmolVM{} + Expect(k8sClient.Get(ctx, name, latest)).To(Succeed()) + Expect(conditionReason(latest, vmv1alpha1.ConditionReconciled)).To(Equal("UnsupportedUpdate")) + Expect(runtime.started).To(Equal(0)) + }) + + It("keeps the finalizer when runtime deletion fails", func() { + ctx := context.Background() + name := types.NamespacedName{Name: "runtime-delete-fail", Namespace: "default"} + machineName := "k8s-default-runtime-delete-fail-test" + Expect(createSmolVMWithStatus(ctx, name, "node-a", machineName)).To(Succeed()) + withNodeName("node-a", func() { + runtime := &fakeRuntime{deleteErr: fmt.Errorf("runtime unavailable")} + Expect(k8sClient.Delete(ctx, &vmv1alpha1.SmolVM{ObjectMeta: metav1.ObjectMeta{Name: name.Name, Namespace: name.Namespace}})).To(Succeed()) + + _, err := testReconciler(runtime).Reconcile(ctx, reconcile.Request{NamespacedName: name}) + Expect(err).NotTo(HaveOccurred()) + Expect(runtime.deleted).To(Equal(1)) + + latest := &vmv1alpha1.SmolVM{} + Expect(k8sClient.Get(ctx, name, latest)).To(Succeed()) + Expect(controllerutil.ContainsFinalizer(latest, vmv1alpha1.SmolVMFinalizer)).To(BeTrue()) + Expect(conditionReason(latest, vmv1alpha1.ConditionReconciled)).To(Equal("RuntimeUnavailable")) + }) + }) + + It("deletes on the status owner node rather than a changed spec node", func() { + ctx := context.Background() + name := types.NamespacedName{Name: "runtime-delete-owner", Namespace: "default"} + machineName := "k8s-default-runtime-delete-owner-test" + vm := testSmolVM(name, "node-b") + controllerutil.AddFinalizer(vm, vmv1alpha1.SmolVMFinalizer) + Expect(k8sClient.Create(ctx, vm)).To(Succeed()) + latest := &vmv1alpha1.SmolVM{} + Expect(k8sClient.Get(ctx, name, latest)).To(Succeed()) + latest.Status.NodeName = "node-a" + latest.Status.MachineName = machineName + Expect(k8sClient.Status().Update(ctx, latest)).To(Succeed()) + + withNodeName("node-a", func() { + runtime := &fakeRuntime{} + Expect(k8sClient.Delete(ctx, &vmv1alpha1.SmolVM{ObjectMeta: metav1.ObjectMeta{Name: name.Name, Namespace: name.Namespace}})).To(Succeed()) + _, err := testReconciler(runtime).Reconcile(ctx, reconcile.Request{NamespacedName: name}) + Expect(err).NotTo(HaveOccurred()) + Expect(runtime.deleted).To(Equal(1)) + }) + }) + + It("starts and stops machines to match desired running state", func() { + ctx := context.Background() + startName := types.NamespacedName{Name: "runtime-start", Namespace: "default"} + Expect(createSmolVMWithStatus(ctx, startName, "node-a", "runtime-start-machine")).To(Succeed()) + startRuntime := &fakeRuntime{machine: &smolvmapi.MachineInfo{Name: "runtime-start-machine", State: "stopped", CPUs: 1, MemoryMiB: 128}} + _, err := testReconciler(startRuntime).Reconcile(ctx, reconcile.Request{NamespacedName: startName}) + Expect(err).NotTo(HaveOccurred()) + Expect(startRuntime.started).To(Equal(1)) + + stopName := types.NamespacedName{Name: "runtime-stop", Namespace: "default"} + vm := testSmolVM(stopName, "node-a") + vm.Spec.Running = false + Expect(createSmolVMWithSpecStatus(ctx, vm, "node-a", "runtime-stop-machine")).To(Succeed()) + stopRuntime := &fakeRuntime{machine: &smolvmapi.MachineInfo{Name: "runtime-stop-machine", State: "running", CPUs: 1, MemoryMiB: 128}} + _, err = testReconciler(stopRuntime).Reconcile(ctx, reconcile.Request{NamespacedName: stopName}) + Expect(err).NotTo(HaveOccurred()) + Expect(stopRuntime.stopped).To(Equal(1)) + }) + + It("removes finalizers after successful or already-missing runtime deletion", func() { + ctx := context.Background() + for _, tc := range []struct { + name string + machine *smolvmapi.MachineInfo + }{ + {name: "runtime-delete-success", machine: &smolvmapi.MachineInfo{Name: "runtime-delete-success-machine", State: "stopped", CPUs: 1, MemoryMiB: 128}}, + {name: "runtime-delete-notfound"}, + } { + name := types.NamespacedName{Name: tc.name, Namespace: "default"} + machineName := tc.name + "-machine" + Expect(createSmolVMWithStatus(ctx, name, "node-a", machineName)).To(Succeed()) + if tc.machine != nil { + tc.machine.Name = machineName + } + runtime := &fakeRuntime{machine: tc.machine} + Expect(k8sClient.Delete(ctx, &vmv1alpha1.SmolVM{ObjectMeta: metav1.ObjectMeta{Name: name.Name, Namespace: name.Namespace}})).To(Succeed()) + _, err := testReconciler(runtime).Reconcile(ctx, reconcile.Request{NamespacedName: name}) + Expect(err).NotTo(HaveOccurred()) + latest := &vmv1alpha1.SmolVM{} + Eventually(func() bool { + err := k8sClient.Get(ctx, name, latest) + return apierrors.IsNotFound(err) + }).Should(BeTrue()) + } + }) + + It("handles storage resize lifecycle", func() { + ctx := context.Background() + expandName := types.NamespacedName{Name: "runtime-storage-expand", Namespace: "default"} + expandVM := testSmolVM(expandName, "node-a") + expandVM.Spec.Storage.StorageGiB = 4 + Expect(createSmolVMWithSpecStatus(ctx, expandVM, "node-a", "runtime-storage-expand-machine")).To(Succeed()) + oldSize := int64(2) + expandRuntime := &fakeRuntime{machine: &smolvmapi.MachineInfo{Name: "runtime-storage-expand-machine", State: "stopped", CPUs: 1, MemoryMiB: 128, StorageGB: &oldSize}} + _, err := testReconciler(expandRuntime).Reconcile(ctx, reconcile.Request{NamespacedName: expandName}) + Expect(err).NotTo(HaveOccurred()) + Expect(expandRuntime.resized).To(Equal(1)) + + shrinkName := types.NamespacedName{Name: "runtime-storage-shrink", Namespace: "default"} + shrinkVM := testSmolVM(shrinkName, "node-a") + shrinkVM.Spec.Storage.StorageGiB = 1 + Expect(createSmolVMWithSpecStatus(ctx, shrinkVM, "node-a", "runtime-storage-shrink-machine")).To(Succeed()) + largeSize := int64(2) + shrinkRuntime := &fakeRuntime{machine: &smolvmapi.MachineInfo{Name: "runtime-storage-shrink-machine", State: "stopped", CPUs: 1, MemoryMiB: 128, StorageGB: &largeSize}} + _, err = testReconciler(shrinkRuntime).Reconcile(ctx, reconcile.Request{NamespacedName: shrinkName}) + Expect(err).NotTo(HaveOccurred()) + Expect(shrinkRuntime.resized).To(Equal(0)) + latest := &vmv1alpha1.SmolVM{} + Expect(k8sClient.Get(ctx, shrinkName, latest)).To(Succeed()) + Expect(conditionReason(latest, vmv1alpha1.ConditionReconciled)).To(Equal("InvalidStorageResize")) + + runningName := types.NamespacedName{Name: "runtime-storage-running", Namespace: "default"} + runningVM := testSmolVM(runningName, "node-a") + runningVM.Spec.Storage.StorageGiB = 4 + Expect(createSmolVMWithSpecStatus(ctx, runningVM, "node-a", "runtime-storage-running-machine")).To(Succeed()) + runningRuntime := &fakeRuntime{machine: &smolvmapi.MachineInfo{Name: "runtime-storage-running-machine", State: "running", CPUs: 1, MemoryMiB: 128, StorageGB: &oldSize}} + _, err = testReconciler(runningRuntime).Reconcile(ctx, reconcile.Request{NamespacedName: runningName}) + Expect(err).NotTo(HaveOccurred()) + Expect(runningRuntime.resized).To(Equal(0)) + Expect(k8sClient.Get(ctx, runningName, latest)).To(Succeed()) + Expect(conditionReason(latest, vmv1alpha1.ConditionReconciled)).To(Equal("ResizeRequiresStoppedMachine")) + }) + + It("marks runtime failures for create, start, stop, and resize", func() { + ctx := context.Background() + cases := []struct { + name string + rt *fakeRuntime + vm *vmv1alpha1.SmolVM + assert func(*fakeRuntime) + }{ + {name: "runtime-create-error", rt: &fakeRuntime{createErr: fmt.Errorf("create failed")}, vm: testSmolVM(types.NamespacedName{Name: "runtime-create-error", Namespace: "default"}, "node-a"), assert: func(rt *fakeRuntime) { Expect(rt.created).To(Equal(1)) }}, + {name: "runtime-start-error", rt: &fakeRuntime{machine: &smolvmapi.MachineInfo{Name: "runtime-start-error-machine", State: "stopped", CPUs: 1, MemoryMiB: 128}, startErr: fmt.Errorf("start failed")}, vm: testSmolVM(types.NamespacedName{Name: "runtime-start-error", Namespace: "default"}, "node-a"), assert: func(rt *fakeRuntime) { Expect(rt.started).To(Equal(1)) }}, + {name: "runtime-stop-error", rt: &fakeRuntime{machine: &smolvmapi.MachineInfo{Name: "runtime-stop-error-machine", State: "running", CPUs: 1, MemoryMiB: 128}, stopErr: fmt.Errorf("stop failed")}, vm: stoppedSmolVM(types.NamespacedName{Name: "runtime-stop-error", Namespace: "default"}, "node-a"), assert: func(rt *fakeRuntime) { Expect(rt.stopped).To(Equal(1)) }}, + {name: "runtime-resize-error", rt: &fakeRuntime{machine: &smolvmapi.MachineInfo{Name: "runtime-resize-error-machine", State: "stopped", CPUs: 1, MemoryMiB: 128, StorageGB: int64Ptr(1)}, resizeErr: fmt.Errorf("resize failed")}, vm: storageSmolVM(types.NamespacedName{Name: "runtime-resize-error", Namespace: "default"}, "node-a", 2), assert: func(rt *fakeRuntime) { Expect(rt.resized).To(Equal(1)) }}, + } + for _, tc := range cases { + name := types.NamespacedName{Name: tc.name, Namespace: "default"} + if tc.name == "runtime-create-error" { + Expect(k8sClient.Create(ctx, tc.vm)).To(Succeed()) + _, err := testReconciler(tc.rt).Reconcile(ctx, reconcile.Request{NamespacedName: name}) + Expect(err).NotTo(HaveOccurred()) + _, err = testReconciler(tc.rt).Reconcile(ctx, reconcile.Request{NamespacedName: name}) + Expect(err).NotTo(HaveOccurred()) + } else { + Expect(createSmolVMWithSpecStatus(ctx, tc.vm, "node-a", tc.name+"-machine")).To(Succeed()) + _, err := testReconciler(tc.rt).Reconcile(ctx, reconcile.Request{NamespacedName: name}) + Expect(err).NotTo(HaveOccurred()) + } + tc.assert(tc.rt) + latest := &vmv1alpha1.SmolVM{} + Expect(k8sClient.Get(ctx, name, latest)).To(Succeed()) + Expect(conditionReason(latest, vmv1alpha1.ConditionReconciled)).To(Equal("RuntimeUnavailable")) + } + }) + + It("records runtime fields and readiness conditions", func() { + ctx := context.Background() + name := types.NamespacedName{Name: "runtime-status-fields", Namespace: "default"} + machineName := "runtime-status-fields-machine" + Expect(createSmolVMWithStatus(ctx, name, "node-a", machineName)).To(Succeed()) + pid := int32(1234) + storage := int64(4) + runtime := &fakeRuntime{machine: &smolvmapi.MachineInfo{Name: machineName, State: "running", CPUs: 1, MemoryMiB: 128, PID: &pid, Network: true, Ports: []smolvmapi.PortSpec{{Host: 8080, Guest: 80}}, StorageGB: &storage}} + _, err := testReconciler(runtime).Reconcile(ctx, reconcile.Request{NamespacedName: name}) + Expect(err).NotTo(HaveOccurred()) + latest := &vmv1alpha1.SmolVM{} + Expect(k8sClient.Get(ctx, name, latest)).To(Succeed()) + Expect(*latest.Status.RuntimePID).To(Equal(pid)) + Expect(latest.Status.Network).To(BeTrue()) + Expect(latest.Status.Ports).To(Equal([]vmv1alpha1.SmolVMPort{{HostPort: 8080, GuestPort: 80}})) + Expect(*latest.Status.StorageGiB).To(Equal(storage)) + Expect(conditionReason(latest, vmv1alpha1.ConditionRuntimeReady)).To(Equal("Observed")) + Expect(conditionReason(latest, vmv1alpha1.ConditionGuestReady)).To(Equal("GuestReadinessUnavailable")) + }) + + It("does not write status when observed state is unchanged", func() { + ctx := context.Background() + name := types.NamespacedName{Name: "runtime-status-unchanged", Namespace: "default"} + machineName := "runtime-status-unchanged-machine" + Expect(createSmolVMWithStatus(ctx, name, "node-a", machineName)).To(Succeed()) + runtime := &fakeRuntime{machine: &smolvmapi.MachineInfo{Name: machineName, State: "running", CPUs: 1, MemoryMiB: 128}} + reconciler := testReconciler(runtime) + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: name}) + Expect(err).NotTo(HaveOccurred()) + latest := &vmv1alpha1.SmolVM{} + Expect(k8sClient.Get(ctx, name, latest)).To(Succeed()) + resourceVersion := latest.ResourceVersion + _, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: name}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient.Get(ctx, name, latest)).To(Succeed()) + Expect(latest.ResourceVersion).To(Equal(resourceVersion)) + }) + + It("no-ops on other nodes and reports awaiting node assignment", func() { + ctx := context.Background() + mismatchName := types.NamespacedName{Name: "runtime-node-mismatch", Namespace: "default"} + Expect(createSmolVMWithStatus(ctx, mismatchName, "node-a", "runtime-node-mismatch-machine")).To(Succeed()) + withNodeName("node-b", func() { + runtime := &fakeRuntime{} + _, err := testReconciler(runtime).Reconcile(ctx, reconcile.Request{NamespacedName: mismatchName}) + Expect(err).NotTo(HaveOccurred()) + Expect(runtime.created + runtime.started + runtime.deleted).To(Equal(0)) + }) + + pendingName := types.NamespacedName{Name: "runtime-awaiting-node", Namespace: "default"} + Expect(k8sClient.Create(ctx, testSmolVM(pendingName, ""))).To(Succeed()) + withNodeName("node-a", func() { + _, err := testReconciler(&fakeRuntime{}).Reconcile(ctx, reconcile.Request{NamespacedName: pendingName}) + Expect(err).NotTo(HaveOccurred()) + }) + latest := &vmv1alpha1.SmolVM{} + Expect(k8sClient.Get(ctx, pendingName, latest)).To(Succeed()) + Expect(conditionReason(latest, vmv1alpha1.ConditionReconciled)).To(Equal("AwaitingNodeAssignment")) + }) + + It("rejects invalid specs through CRD admission", func() { + ctx := context.Background() + both := testSmolVM(types.NamespacedName{Name: "admission-image-from", Namespace: "default"}, "node-a") + both.Spec.From = "/tmp/machine.smolmachine" + Expect(k8sClient.Create(ctx, both)).NotTo(Succeed()) + + duplicatePorts := testSmolVM(types.NamespacedName{Name: "admission-duplicate-ports", Namespace: "default"}, "node-a") + duplicatePorts.Spec.Network.Ports = []vmv1alpha1.SmolVMPort{{HostPort: 8080, GuestPort: 80}, {HostPort: 8080, GuestPort: 81}} + Expect(k8sClient.Create(ctx, duplicatePorts)).NotTo(Succeed()) + + invalidCIDR := testSmolVM(types.NamespacedName{Name: "admission-invalid-cidr", Namespace: "default"}, "node-a") + invalidCIDR.Spec.Network.AllowedCIDRs = []string{"not-a-cidr"} + Expect(k8sClient.Create(ctx, invalidCIDR)).NotTo(Succeed()) + }) + + It("emits Kubernetes events for lifecycle and failure transitions", func() { + ctx := context.Background() + createName := types.NamespacedName{Name: "event-create", Namespace: "default"} + Expect(k8sClient.Create(ctx, testSmolVM(createName, "node-a"))).To(Succeed()) + createRecorder := record.NewFakeRecorder(4) + createRuntime := &fakeRuntime{} + _, err := testReconcilerWithRecorder(createRuntime, createRecorder).Reconcile(ctx, reconcile.Request{NamespacedName: createName}) + Expect(err).NotTo(HaveOccurred()) + _, err = testReconcilerWithRecorder(createRuntime, createRecorder).Reconcile(ctx, reconcile.Request{NamespacedName: createName}) + Expect(err).NotTo(HaveOccurred()) + Eventually(createRecorder.Events).Should(Receive(ContainSubstring("Creating"))) + + failureName := types.NamespacedName{Name: "event-runtime-failure", Namespace: "default"} + Expect(createSmolVMWithStatus(ctx, failureName, "node-a", "event-runtime-failure-machine")).To(Succeed()) + failureRecorder := record.NewFakeRecorder(4) + failureRuntime := &fakeRuntime{getErr: fmt.Errorf("runtime unavailable")} + _, err = testReconcilerWithRecorder(failureRuntime, failureRecorder).Reconcile(ctx, reconcile.Request{NamespacedName: failureName}) + Expect(err).NotTo(HaveOccurred()) + Eventually(failureRecorder.Events).Should(Receive(ContainSubstring("RuntimeUnavailable"))) + }) }) +func testSmolVM(name types.NamespacedName, nodeName string) *vmv1alpha1.SmolVM { + return &vmv1alpha1.SmolVM{ + ObjectMeta: metav1.ObjectMeta{Name: name.Name, Namespace: name.Namespace}, + Spec: vmv1alpha1.SmolVMSpec{ + Running: true, + NodeName: nodeName, + Image: "alpine:latest", + Resources: vmv1alpha1.SmolVMResources{ + CPUs: 1, + MemoryMiB: 128, + }, + }, + } +} + +func stoppedSmolVM(name types.NamespacedName, nodeName string) *vmv1alpha1.SmolVM { + vm := testSmolVM(name, nodeName) + vm.Spec.Running = false + return vm +} + +func storageSmolVM(name types.NamespacedName, nodeName string, storageGiB int64) *vmv1alpha1.SmolVM { + vm := testSmolVM(name, nodeName) + vm.Spec.Storage.StorageGiB = storageGiB + return vm +} + +func int64Ptr(value int64) *int64 { + return &value +} + +func testReconciler(runtime *fakeRuntime) *SmolVMReconciler { + return testReconcilerWithRecorder(runtime, nil) +} + +func testReconcilerWithRecorder(runtime *fakeRuntime, recorder record.EventRecorder) *SmolVMReconciler { + return &SmolVMReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Recorder: recorder, + RuntimeFactory: func() SmolVMRuntime { + return runtime + }, + } +} + +func createSmolVMWithStatus(ctx context.Context, name types.NamespacedName, nodeName, machineName string) error { + return createSmolVMWithSpecStatus(ctx, testSmolVM(name, nodeName), nodeName, machineName) +} + +func createSmolVMWithSpecStatus(ctx context.Context, vm *vmv1alpha1.SmolVM, nodeName, machineName string) error { + name := types.NamespacedName{Name: vm.Name, Namespace: vm.Namespace} + controllerutil.AddFinalizer(vm, vmv1alpha1.SmolVMFinalizer) + if err := k8sClient.Create(ctx, vm); err != nil { + return err + } + latest := &vmv1alpha1.SmolVM{} + if err := k8sClient.Get(ctx, name, latest); err != nil { + return err + } + latest.Status.NodeName = nodeName + latest.Status.MachineName = machineName + return k8sClient.Status().Update(ctx, latest) +} + +func withNodeName(nodeName string, fn func()) { + previous, hadPrevious := os.LookupEnv("SMOLVM_NODE_NAME") + Expect(os.Setenv("SMOLVM_NODE_NAME", nodeName)).To(Succeed()) + defer func() { + if hadPrevious { + Expect(os.Setenv("SMOLVM_NODE_NAME", previous)).To(Succeed()) + return + } + Expect(os.Unsetenv("SMOLVM_NODE_NAME")).To(Succeed()) + }() + fn() +} + +func conditionReason(vm *vmv1alpha1.SmolVM, conditionType string) string { + for _, condition := range vm.Status.Conditions { + if condition.Type == conditionType { + return condition.Reason + } + } + return "" +} + type fakeRuntime struct { - created int + machine *smolvmapi.MachineInfo + getErr error + createErr error + startErr error + stopErr error + deleteErr error + resizeErr error + created int + started int + stopped int + deleted int + resized int } func (f *fakeRuntime) GetMachine(context.Context, string) (*smolvmapi.MachineInfo, error) { - return nil, smolvmapi.APIError{StatusCode: http.StatusNotFound} + if f.getErr != nil { + return nil, f.getErr + } + if f.machine == nil { + return nil, smolvmapi.APIError{StatusCode: http.StatusNotFound} + } + return f.machine, nil } func (f *fakeRuntime) CreateMachine(_ context.Context, req smolvmapi.CreateMachineRequest) (*smolvmapi.MachineInfo, error) { f.created++ - return &smolvmapi.MachineInfo{ + if f.createErr != nil { + return nil, f.createErr + } + f.machine = &smolvmapi.MachineInfo{ Name: req.Name, State: "stopped", CPUs: req.CPUs, MemoryMiB: req.MemoryMiB, - }, nil + Ports: req.Ports, + Network: req.Network, + StorageGB: req.StorageGB, + OverlayGB: req.OverlayGB, + } + return f.machine, nil } -func (f *fakeRuntime) EnsureMachineRunning(context.Context, string) error { return nil } -func (f *fakeRuntime) StopMachine(context.Context, string) error { return nil } -func (f *fakeRuntime) DeleteMachine(context.Context, string) error { return nil } -func (f *fakeRuntime) ResizeMachine(context.Context, string, smolvmapi.ResizeRequest) error { +func (f *fakeRuntime) EnsureMachineRunning(context.Context, string) error { + f.started++ + if f.startErr != nil { + return f.startErr + } + if f.machine != nil { + f.machine.State = "running" + } + return nil +} + +func (f *fakeRuntime) StopMachine(context.Context, string) error { + f.stopped++ + if f.stopErr != nil { + return f.stopErr + } + if f.machine != nil { + f.machine.State = "stopped" + } + return nil +} + +func (f *fakeRuntime) DeleteMachine(context.Context, string) error { + f.deleted++ + if f.deleteErr != nil { + return f.deleteErr + } + f.machine = nil return nil } + +func (f *fakeRuntime) ResizeMachine(context.Context, string, smolvmapi.ResizeRequest) error { + f.resized++ + return f.resizeErr +} diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 79710f0..ee72ae2 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -40,12 +40,27 @@ var _ = Describe("controller", Ordered, func() { _, err = utils.Run(exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectImage))) Expect(err).NotTo(HaveOccurred()) - By("waiting for the controller deployment") - _, err = utils.Run(exec.Command("kubectl", "wait", "deployment", "operator-controller-manager", "-n", namespace, "--for=condition=Available", "--timeout=2m")) + By("waiting for the controller daemonset") + _, err = utils.Run(exec.Command("kubectl", "rollout", "status", "daemonset/operator-controller-manager", "-n", namespace, "--timeout=2m")) Expect(err).NotTo(HaveOccurred()) - By("applying a SmolVM resource") - _, err = utils.Run(exec.Command("kubectl", "apply", "-f", "config/samples/vm_v1alpha1_smolvm.yaml")) + By("applying a node-pinned SmolVM resource") + _, err = utils.Run(exec.Command("bash", "-c", ` +node=$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}') +cat <