diff --git a/ROADMAP.md b/ROADMAP.md index 5f27ed4..acfdbea 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -168,10 +168,20 @@ prerequisite (Longhorn); the rest are convenience + robustness. omitempty, special-char round-trip), spec parse, and a handler wiring test asserting the per-VM upload + cicustom. *Metal validation (does open-iscsi actually install) is a homelab follow-up, like A1–A4.* -- [ ] **E2 — k3s node-pool node prep.** Surface E1 through the k3s Cluster - spec (a per-pool `nodePrep`/`packages`/`runcmd` block) so a worker pool - installs its own host deps, composed into the node VMs (mirrors the - GPU-per-pool shape from A3). +- [x] **E2 — k3s node-pool node prep.** The k3s Cluster spec now takes a + per-pool `nodePrep: {packages, runcmd}` block (on control-plane and each + worker pool), stamped onto that pool's node VMs' `cloudInit` (which the + Proxmox provider renders via E1). Mirrors the GPU-per-pool shape from A3 + exactly: `NodePrepSpec` + `parseNodePrepSpec` + `NodePrepForNode(i,cpCount)` + + `ApplyNodePrepToVMSpec` (writes into the existing `cloudInit` map, + preserving user/sshKeys/ipConfig), wired into **both** VM-build paths + (operative `create.go` + Plan mirror `cluster_plan.go`) via the shared + helper. `#NodePrep` CUE def on both pools. So a worker pool installs its + own host deps — e.g. `open-iscsi` for a Longhorn storage pool. Tests: + parse+resolve across the flat node ordering, apply-into-cloudInit + (nil/empty no-op, preserves siblings), and an operative-path + `GenerateDispatchRequests` test asserting only the target pool's VMs get + the prereqs. **Makes F1 (Longhorn) usable.** ### F. Storage - [ ] **F1 — Longhorn Platform component.** Add `longhorn` to the opt-in diff --git a/internal/controller/providers/k3s/cluster_plan.go b/internal/controller/providers/k3s/cluster_plan.go index 7376aca..c45f2f1 100644 --- a/internal/controller/providers/k3s/cluster_plan.go +++ b/internal/controller/providers/k3s/cluster_plan.go @@ -237,6 +237,9 @@ func buildVMManifest(clusterName, nodeName string, i, cpCount int, size k3sresou // GPU/PCI passthrough for this node's pool — kept identical to the // operative create.go path via the shared resources helpers. k3sresources.ApplyGPUToVMSpec(vm.Spec, k3sresources.GPUForNode(i, cpCount, spec)) + // Host prerequisites (cloud-init packages/runcmd) for this node's pool — + // same shared helper as create.go so both paths emit identical VMs. + k3sresources.ApplyNodePrepToVMSpec(vm.Spec, k3sresources.NodePrepForNode(i, cpCount, spec)) return vm } diff --git a/internal/schema/schemas/k3s/cluster.cue b/internal/schema/schemas/k3s/cluster.cue index e5297ef..1e07c3c 100644 --- a/internal/schema/schemas/k3s/cluster.cue +++ b/internal/schema/schemas/k3s/cluster.cue @@ -96,6 +96,16 @@ import "openctl.io/schemas/base" cpuType?: string | *"host" } +// #NodePrep installs host prerequisites on every node in a pool via the node +// VM's cloud-init — e.g. "open-iscsi" for a Longhorn storage pool. Rendered +// into a cloud-init vendor snippet by the Proxmox provider. +#NodePrep: { + // Host packages installed on first boot (package index refreshed first). + packages?: [...string] + // First-boot shell commands, run after qemu-guest-agent is enabled. + runcmd?: [...string] +} + #ClusterSpec: { // Which infrastructure provider runs the VMs. Currently only // "proxmox" is implemented; other providers may follow. @@ -152,6 +162,9 @@ import "openctl.io/schemas/base" // GPU/PCI passthrough for the control-plane VMs. Rare (GPUs usually // belong on workers) but supported for symmetry. gpu?: #GPU + // Host prerequisites (packages/runcmd) installed on the + // control-plane VMs via cloud-init. + nodePrep?: #NodePrep } // Worker (agent) node pools. Each pool can have its own size. workers?: [...{ @@ -172,6 +185,9 @@ import "openctl.io/schemas/base" // GPU/PCI passthrough for every node in this pool. Pin the pool to // the host(s) with the device via nodes/targets. gpu?: #GPU + // Host prerequisites (packages/runcmd) installed on every node in + // this pool via cloud-init — e.g. open-iscsi for a Longhorn pool. + nodePrep?: #NodePrep }] } diff --git a/pkg/k3s/cluster/create.go b/pkg/k3s/cluster/create.go index 207d43a..2b6adf0 100644 --- a/pkg/k3s/cluster/create.go +++ b/pkg/k3s/cluster/create.go @@ -139,6 +139,10 @@ func (c *Creator) GenerateDispatchRequests() []*protocol.DispatchRequest { // cpu host + hostpci). No-op when the pool requests no GPU. resources.ApplyGPUToVMSpec(manifest.Spec, resources.GPUForNode(i, len(cpNodes), c.spec)) + // Host prerequisites for this node's pool (cloud-init packages/runcmd, + // e.g. open-iscsi for Longhorn). No-op when the pool requests none. + resources.ApplyNodePrepToVMSpec(manifest.Spec, resources.NodePrepForNode(i, len(cpNodes), c.spec)) + // Place this VM on a specific provider endpoint/host when the // pool defines placement; otherwise leave spec.context/spec.node // unset so the provider uses its configured defaults. diff --git a/pkg/k3s/cluster/create_placement_test.go b/pkg/k3s/cluster/create_placement_test.go index 2360147..dfbfd50 100644 --- a/pkg/k3s/cluster/create_placement_test.go +++ b/pkg/k3s/cluster/create_placement_test.go @@ -187,3 +187,66 @@ func TestGenerateDispatchRequests_GPU(t *testing.T) { } } } + +// TestGenerateDispatchRequests_NodePrep verifies a per-pool nodePrep block +// stamps cloud-init packages/runcmd onto only that pool's node VMs (via the +// operative create path), leaving other pools' cloudInit untouched. +func TestGenerateDispatchRequests_NodePrep(t *testing.T) { + spec := &resources.ClusterSpec{ + Compute: resources.ComputeSpec{ + Provider: "proxmox", + Image: resources.ImageSpec{Template: "ubuntu-template"}, + Default: resources.DefaultSizeSpec{CPUs: 2, MemoryMB: 4096, DiskGB: 40}, + }, + Nodes: resources.NodesSpec{ + ControlPlane: resources.ControlPlaneSpec{Count: 1}, + Workers: []resources.WorkerSpec{ + {Name: "general", Count: 1}, + {Name: "storage", Count: 1, NodePrep: &resources.NodePrepSpec{ + Packages: []string{"open-iscsi"}, + RunCmd: []string{"systemctl enable iscsid"}, + }}, + }, + }, + SSH: resources.SSHSpec{User: "ubuntu"}, + } + + byID := map[string]*protocol.Resource{} + for _, req := range NewCreator("dev", spec, &protocol.ProviderConfig{}).GenerateDispatchRequests() { + byID[req.ID] = req.Manifest + } + + // The storage worker's cloudInit carries the prereqs, preserving user. + storage := byID["vm-dev-storage-0"] + if storage == nil { + t.Fatalf("missing storage node request; got %v", byID) + } + ci, _ := storage.Spec["cloudInit"].(map[string]any) + if ci == nil || ci["user"] != "ubuntu" { + t.Fatalf("storage cloudInit missing/clobbered: %v", storage.Spec["cloudInit"]) + } + if pkgs, _ := ci["packages"].([]string); len(pkgs) != 1 || pkgs[0] != "open-iscsi" { + t.Errorf("storage packages wrong: %v", ci["packages"]) + } + if cmds, _ := ci["runcmd"].([]string); len(cmds) != 1 || cmds[0] != "systemctl enable iscsid" { + t.Errorf("storage runcmd wrong: %v", ci["runcmd"]) + } + + // The general worker and control plane must NOT get packages/runcmd. + for _, id := range []string{"vm-dev-cp-0", "vm-dev-general-0"} { + m := byID[id] + if m == nil { + t.Fatalf("missing %s", id) + } + ci, _ := m.Spec["cloudInit"].(map[string]any) + if ci == nil { + t.Fatalf("%s missing cloudInit", id) + } + if _, ok := ci["packages"]; ok { + t.Errorf("%s should not have cloudInit.packages", id) + } + if _, ok := ci["runcmd"]; ok { + t.Errorf("%s should not have cloudInit.runcmd", id) + } + } +} diff --git a/pkg/k3s/resources/cluster.go b/pkg/k3s/resources/cluster.go index 9112178..e8e42f6 100644 --- a/pkg/k3s/resources/cluster.go +++ b/pkg/k3s/resources/cluster.go @@ -120,6 +120,20 @@ type PCIDevice struct { MDev string `json:"mdev,omitempty"` } +// NodePrepSpec installs host prerequisites on every node in a pool via the +// underlying VM's cloud-init (packages + first-boot commands). Use it for node +// dependencies that must exist before workloads land — e.g. `open-iscsi` for +// Longhorn. Stamped onto each node VM's `cloudInit` block, which the Proxmox +// provider renders into a cloud-init vendor snippet (see E1). Proxmox-specific. +type NodePrepSpec struct { + // Packages are host packages installed on first boot (with the package + // index refreshed first). + Packages []string `json:"packages,omitempty"` + // RunCmd are first-boot shell commands, run after the node's + // qemu-guest-agent enablement. + RunCmd []string `json:"runcmd,omitempty"` +} + // NodesSpec defines the cluster nodes type NodesSpec struct { ControlPlane ControlPlaneSpec `json:"controlPlane"` @@ -146,6 +160,9 @@ type ControlPlaneSpec struct { // GPU requests PCI/GPU passthrough for the control-plane VMs. Rare — GPUs // usually belong on workers — but supported for symmetry. GPU *GPUSpec `json:"gpu,omitempty"` + // NodePrep installs host prerequisites (packages/runcmd) on the + // control-plane VMs via cloud-init. + NodePrep *NodePrepSpec `json:"nodePrep,omitempty"` } // WorkerSpec defines a worker node pool @@ -165,6 +182,9 @@ type WorkerSpec struct { // pool of one node that runs a local model). Pin the pool to the host(s) // with the device via Nodes/Targets. GPU *GPUSpec `json:"gpu,omitempty"` + // NodePrep installs host prerequisites (packages/runcmd) on every node in + // this pool via cloud-init — e.g. `open-iscsi` for a Longhorn storage pool. + NodePrep *NodePrepSpec `json:"nodePrep,omitempty"` } // K3sSpec defines K3s configuration @@ -243,6 +263,9 @@ func ParseClusterSpec(r *protocol.Resource) (*ClusterSpec, error) { if gpu, ok := cp["gpu"].(map[string]any); ok { spec.Nodes.ControlPlane.GPU = parseGPUSpec(gpu) } + if np, ok := cp["nodePrep"].(map[string]any); ok { + spec.Nodes.ControlPlane.NodePrep = parseNodePrepSpec(np) + } } if workers, ok := nodes["workers"].([]any); ok { for _, w := range workers { @@ -265,6 +288,9 @@ func ParseClusterSpec(r *protocol.Resource) (*ClusterSpec, error) { if gpu, ok := worker["gpu"].(map[string]any); ok { ws.GPU = parseGPUSpec(gpu) } + if np, ok := worker["nodePrep"].(map[string]any); ok { + ws.NodePrep = parseNodePrepSpec(np) + } spec.Nodes.Workers = append(spec.Nodes.Workers, ws) } } @@ -509,6 +535,68 @@ func ApplyGPUToVMSpec(vmSpec map[string]any, gpu *GPUSpec) { } } +// parseNodePrepSpec parses a pool's nodePrep block (packages + runcmd) from the +// untyped manifest map. Returns nil-safe empty slices; callers no-op on empty. +func parseNodePrepSpec(m map[string]any) *NodePrepSpec { + np := &NodePrepSpec{} + if pkgs, ok := m["packages"].([]any); ok { + for _, p := range pkgs { + if pkg, ok := p.(string); ok { + np.Packages = append(np.Packages, pkg) + } + } + } + if cmds, ok := m["runcmd"].([]any); ok { + for _, c := range cmds { + if cmd, ok := c.(string); ok { + np.RunCmd = append(np.RunCmd, cmd) + } + } + } + return np +} + +// NodePrepForNode resolves the node-prep config for node index i (across the +// flat control-plane-then-workers ordering NodeNames produces), or nil when the +// node's pool requests none. Mirrors GPUForNode so both VM-build paths stay in +// sync. +func NodePrepForNode(i, cpCount int, spec *ClusterSpec) *NodePrepSpec { + if i < cpCount { + return spec.Nodes.ControlPlane.NodePrep + } + workerIdx := i - cpCount + for _, pool := range spec.Nodes.Workers { + if workerIdx < pool.Count { + return pool.NodePrep + } + workerIdx -= pool.Count + } + return nil +} + +// ApplyNodePrepToVMSpec stamps a pool's host prerequisites onto a node VM's +// cloud-init block (packages + runcmd), which the Proxmox provider renders into +// a cloud-init vendor snippet. No-op when np is nil or carries nothing. Shared +// by both VM-build paths (create.go and the Plan mirror) so nodes come out +// identical. It writes into the existing vmSpec["cloudInit"] map (built by both +// paths just before this call), preserving user/sshKeys/ipConfig. +func ApplyNodePrepToVMSpec(vmSpec map[string]any, np *NodePrepSpec) { + if np == nil || (len(np.Packages) == 0 && len(np.RunCmd) == 0) { + return + } + ci, ok := vmSpec["cloudInit"].(map[string]any) + if !ok { + ci = map[string]any{} + vmSpec["cloudInit"] = ci + } + if len(np.Packages) > 0 { + ci["packages"] = np.Packages + } + if len(np.RunCmd) > 0 { + ci["runcmd"] = np.RunCmd + } +} + // ClusterToResource converts cluster state to a protocol Resource func ClusterToResource(name string, spec *ClusterSpec, phase string, outputs map[string]any, children []protocol.ChildReference) *protocol.Resource { specMap := map[string]any{ diff --git a/pkg/k3s/resources/nodeprep_test.go b/pkg/k3s/resources/nodeprep_test.go new file mode 100644 index 0000000..1364ee0 --- /dev/null +++ b/pkg/k3s/resources/nodeprep_test.go @@ -0,0 +1,99 @@ +package resources + +import ( + "testing" + + "github.com/openctl/openctl/pkg/protocol" +) + +// TestParseAndResolveNodePrep covers parsing a per-pool nodePrep spec and +// resolving it to the right node index across the flat control-plane-then- +// workers ordering — mirrors the GPU resolution both VM-build paths rely on. +func TestParseAndResolveNodePrep(t *testing.T) { + r := &protocol.Resource{Spec: map[string]any{ + "compute": map[string]any{"provider": "proxmox"}, + "nodes": map[string]any{ + "controlPlane": map[string]any{"count": float64(1)}, + "workers": []any{ + map[string]any{"name": "general", "count": float64(2)}, + map[string]any{"name": "storage", "count": float64(1), "nodePrep": map[string]any{ + "packages": []any{"open-iscsi", "nfs-common"}, + "runcmd": []any{"systemctl enable iscsid"}, + }}, + }, + }, + }} + + spec, err := ParseClusterSpec(r) + if err != nil { + t.Fatalf("ParseClusterSpec: %v", err) + } + if spec.Nodes.Workers[0].NodePrep != nil { + t.Errorf("general pool should have no nodePrep") + } + np := spec.Nodes.Workers[1].NodePrep + if np == nil || len(np.Packages) != 2 || np.Packages[0] != "open-iscsi" { + t.Fatalf("storage pool nodePrep mis-parsed: %+v", np) + } + if len(np.RunCmd) != 1 || np.RunCmd[0] != "systemctl enable iscsid" { + t.Errorf("runcmd mis-parsed: %+v", np.RunCmd) + } + + // Flat ordering: index 0 = CP, 1-2 = general workers, 3 = storage worker. + cpCount := 1 + if NodePrepForNode(0, cpCount, spec) != nil { + t.Errorf("CP node should resolve to no nodePrep") + } + if NodePrepForNode(1, cpCount, spec) != nil || NodePrepForNode(2, cpCount, spec) != nil { + t.Errorf("general workers should resolve to no nodePrep") + } + if NodePrepForNode(3, cpCount, spec) != np { + t.Errorf("storage worker (index 3) should resolve to the storage pool nodePrep") + } +} + +func TestApplyNodePrepToVMSpec(t *testing.T) { + // Nil is a no-op. + vm := map[string]any{"cloudInit": map[string]any{"user": "ubuntu"}} + ApplyNodePrepToVMSpec(vm, nil) + if _, ok := vm["cloudInit"].(map[string]any)["packages"]; ok { + t.Errorf("nil nodePrep should be a no-op") + } + + // Empty spec is also a no-op (no packages, no runcmd). + ApplyNodePrepToVMSpec(vm, &NodePrepSpec{}) + if _, ok := vm["cloudInit"].(map[string]any)["packages"]; ok { + t.Errorf("empty nodePrep should be a no-op") + } + + // Full stamp writes into the existing cloudInit map, preserving user. + ApplyNodePrepToVMSpec(vm, &NodePrepSpec{ + Packages: []string{"open-iscsi"}, + RunCmd: []string{"systemctl enable iscsid"}, + }) + ci := vm["cloudInit"].(map[string]any) + if ci["user"] != "ubuntu" { + t.Errorf("cloudInit.user should be preserved, got %v", ci["user"]) + } + if pkgs, ok := ci["packages"].([]string); !ok || len(pkgs) != 1 || pkgs[0] != "open-iscsi" { + t.Errorf("packages not stamped into cloudInit: %v", ci["packages"]) + } + if cmds, ok := ci["runcmd"].([]string); !ok || len(cmds) != 1 || cmds[0] != "systemctl enable iscsid" { + t.Errorf("runcmd not stamped into cloudInit: %v", ci["runcmd"]) + } +} + +// TestApplyNodePrepToVMSpec_CreatesCloudInitWhenAbsent proves the stamp is +// robust to a VM spec that somehow lacks a cloudInit map (both build paths +// construct one first, but the helper shouldn't panic if it's missing). +func TestApplyNodePrepToVMSpec_CreatesCloudInitWhenAbsent(t *testing.T) { + vm := map[string]any{} + ApplyNodePrepToVMSpec(vm, &NodePrepSpec{Packages: []string{"curl"}}) + ci, ok := vm["cloudInit"].(map[string]any) + if !ok { + t.Fatalf("cloudInit should have been created, got %T", vm["cloudInit"]) + } + if pkgs := ci["packages"].([]string); pkgs[0] != "curl" { + t.Errorf("packages = %v", pkgs) + } +}