Skip to content

Commit 1fc6c29

Browse files
authored
feat(proxmox): cloud-init packages + runcmd via per-VM vendor snippet (E1) (#145)
Extend the VirtualMachine cloudInit spec with `packages` (installed on first boot, index refreshed first) and `runcmd` (first-boot commands), so a VM can install node prerequisites like open-iscsi for Longhorn. Proxmox has no native ci* option for arbitrary packages/runcmd, so these render into a per-VM cloud-init vendor snippet (client.RenderVendorData, via yaml.Marshal for safe escaping of arbitrary commands) uploaded to snippets storage and attached with `cicustom vendor=`. vendor-data is additive to Proxmox's generated user-data — unlike a `user=` snippet, which would replace ciuser/cipassword/sshkeys — so those are preserved. The no-packages/no-runcmd path keeps the shared static qemu-agent snippet byte-identical (no risk to the metal-validated QGA IP-discovery flow); the combined path folds the agent runcmd in so IP discovery still works. Both proxmox create paths (template clone, cloud-image) share one helper. Also documents the previously-missing `packageUpgrade` in the CUE schema. plugins/proxmox gains a yaml.v3 indirect require + go.sum (transitive via pkg/proxmox/client). Unblocks F1 (Longhorn). Tests: vendor-data render (ordering, omitempty, special-char round-trip), spec parse, handler wiring (per-VM upload + cicustom). Metal validation of actual package install is a homelab follow-up, like A1-A4.
1 parent bc3d8ae commit 1fc6c29

10 files changed

Lines changed: 448 additions & 29 deletions

File tree

ROADMAP.md

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -151,11 +151,23 @@ wiring). These targets close the remaining gaps. Only **E1** is a genuine
151151
prerequisite (Longhorn); the rest are convenience + robustness.
152152

153153
### E. Node customization
154-
- [ ] **E1 — cloud-init `packages` + `runcmd`.** Extend the Proxmox VM
155-
`cloudInit` spec (today user/ssh/network/packageUpgrade only) to install
156-
host packages and run first-boot commands. Unblocks node-level
157-
prerequisites — e.g. `open-iscsi` for Longhorn — and general node tuning.
158-
*The one true prerequisite in this list.*
154+
- [x] **E1 — cloud-init `packages` + `runcmd`.** The Proxmox VM `cloudInit`
155+
spec now takes `packages: [...string]` (installed on first boot with a
156+
preceding index refresh — `package_update: true`) and `runcmd: [...string]`
157+
(first-boot commands, after the qemu-guest-agent enablement). Rendered
158+
into a **per-VM cloud-init vendor snippet** (`client.RenderVendorData` via
159+
`yaml.Marshal` so arbitrary commands escape safely) uploaded to snippets
160+
storage and attached via `cicustom vendor=` — Proxmox has no native `ci*`
161+
option for arbitrary packages, and vendor-data is additive (doesn't clobber
162+
the generated ciuser/sshkeys user-data that a `user=` snippet would). The
163+
no-packages path keeps the shared static qemu-agent snippet **byte-
164+
identical** (zero risk to the metal-validated QGA flow); the combined path
165+
folds the agent runcmd in so IP discovery still works. Also added the
166+
previously-undocumented `packageUpgrade` to the CUE schema. Unblocks F1
167+
(Longhorn needs `open-iscsi`). Tests: render (packages/runcmd ordering,
168+
omitempty, special-char round-trip), spec parse, and a handler wiring test
169+
asserting the per-VM upload + cicustom. *Metal validation (does open-iscsi
170+
actually install) is a homelab follow-up, like A1–A4.*
159171
- [ ] **E2 — k3s node-pool node prep.** Surface E1 through the k3s Cluster
160172
spec (a per-pool `nodePrep`/`packages`/`runcmd` block) so a worker pool
161173
installs its own host deps, composed into the node VMs (mirrors the

internal/schema/schemas/proxmox/vm.cue

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -215,6 +215,20 @@ import "openctl.io/schemas/base"
215215
searchDomain?: string
216216
// DNS resolver addresses to write to /etc/resolv.conf.
217217
nameservers?: [...string]
218+
// Run a full package upgrade (`apt dist-upgrade`) on first boot
219+
// (Proxmox `ciupgrade`). Defaults OFF so clones boot fast and
220+
// deterministically — a slow upgrade over a flaky link can wedge
221+
// cloud-init. Requires Proxmox VE 8.2+. Set true to opt back in.
222+
packageUpgrade?: bool
223+
// Host packages to install on first boot (cloud-init `packages:`,
224+
// with the index refreshed first). For node prerequisites such as
225+
// "open-iscsi" (Longhorn). Rendered into a per-VM cloud-init vendor
226+
// snippet — Proxmox has no native option for arbitrary packages.
227+
packages?: [...string]
228+
// First-boot shell commands (cloud-init `runcmd:`), run after the
229+
// qemu-guest-agent enablement commands. General node tuning /
230+
// prerequisite hooks. Rendered into the same vendor snippet as packages.
231+
runcmd?: [...string]
218232
// Per-interface IP configuration. Key is the interface name (e.g.
219233
// "net0"). Use "dhcp" as ip to request DHCP, or "<addr>/<cidr>"
220234
// for static with optional gateway.

pkg/proxmox/client/client.go

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ import (
1717
"sort"
1818
"strings"
1919
"time"
20+
21+
"gopkg.in/yaml.v3"
2022
)
2123

2224
var debugEnabled = os.Getenv("OPENCTL_DEBUG") != ""
@@ -1107,6 +1109,55 @@ runcmd:
11071109
- systemctl start qemu-guest-agent
11081110
`
11091111

1112+
// vendorCloudConfig is the subset of cloud-init vendor-data openctl emits: the
1113+
// qemu-guest-agent enablement commands (openctl relies on the agent for IP
1114+
// discovery) plus optional host packages and first-boot commands. Marshaled
1115+
// via yaml so arbitrary runcmd/package strings are escaped correctly rather
1116+
// than hand-concatenated.
1117+
type vendorCloudConfig struct {
1118+
PackageUpdate bool `yaml:"package_update,omitempty"`
1119+
Packages []string `yaml:"packages,omitempty"`
1120+
RunCmd []string `yaml:"runcmd,omitempty"`
1121+
}
1122+
1123+
// agentRunCmds enables + starts qemu-guest-agent. Prepended to every rendered
1124+
// vendor snippet so a VM with custom packages/runcmd keeps IP discovery — the
1125+
// same commands the shared QemuAgentSnippetContent carries.
1126+
var agentRunCmds = []string{
1127+
"systemctl enable qemu-guest-agent",
1128+
"systemctl start qemu-guest-agent",
1129+
}
1130+
1131+
// RenderVendorData builds a #cloud-config vendor-data snippet enabling
1132+
// qemu-guest-agent, installing any requested packages (with a package index
1133+
// refresh), and running any first-boot commands after the agent commands.
1134+
//
1135+
// Emitted as vendor-data (attached via `cicustom vendor=`), which is ADDITIVE
1136+
// to Proxmox's generated user-data — so ciuser/cipassword/sshkeys/etc. are
1137+
// preserved. A `user=` snippet would REPLACE that generated user-data, which
1138+
// is why openctl never uses the user slot here.
1139+
func RenderVendorData(packages, runcmd []string) (string, error) {
1140+
cfg := vendorCloudConfig{
1141+
Packages: packages,
1142+
RunCmd: append(append([]string{}, agentRunCmds...), runcmd...),
1143+
}
1144+
if len(packages) > 0 {
1145+
cfg.PackageUpdate = true
1146+
}
1147+
out, err := yaml.Marshal(cfg)
1148+
if err != nil {
1149+
return "", fmt.Errorf("marshal vendor cloud-config: %w", err)
1150+
}
1151+
return "#cloud-config\n# Created by openctl\n" + string(out), nil
1152+
}
1153+
1154+
// VendorSnippetName is the per-VM vendor snippet filename, keyed by vmid. Per-VM
1155+
// (unlike the shared static QemuAgentSnippetName) because the content varies
1156+
// with the VM's packages/runcmd.
1157+
func VendorSnippetName(vmid int) string {
1158+
return fmt.Sprintf("openctl-vendor-%d.yaml", vmid)
1159+
}
1160+
11101161
// EnsureQemuAgentSnippet ensures the qemu-agent enablement snippet exists
11111162
func (c *Client) EnsureQemuAgentSnippet(ctx context.Context, node, storage string) error {
11121163
exists, err := c.SnippetExists(ctx, node, storage, QemuAgentSnippetName)
Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
package client
2+
3+
import (
4+
"strings"
5+
"testing"
6+
7+
"gopkg.in/yaml.v3"
8+
)
9+
10+
// parseCloudConfig strips the "#cloud-config" header and unmarshals the rest,
11+
// so tests assert on structured content rather than brittle substring matches.
12+
func parseCloudConfig(t *testing.T, s string) map[string]any {
13+
t.Helper()
14+
if !strings.HasPrefix(s, "#cloud-config\n") {
15+
t.Fatalf("vendor data must start with #cloud-config header, got:\n%s", s)
16+
}
17+
var out map[string]any
18+
if err := yaml.Unmarshal([]byte(s), &out); err != nil {
19+
t.Fatalf("rendered vendor data is not valid YAML: %v\n%s", err, s)
20+
}
21+
return out
22+
}
23+
24+
func asStrings(t *testing.T, v any, field string) []string {
25+
t.Helper()
26+
list, ok := v.([]any)
27+
if !ok {
28+
t.Fatalf("%s: want a list, got %T", field, v)
29+
}
30+
out := make([]string, len(list))
31+
for i, e := range list {
32+
s, ok := e.(string)
33+
if !ok {
34+
t.Fatalf("%s[%d]: want string, got %T", field, i, e)
35+
}
36+
out[i] = s
37+
}
38+
return out
39+
}
40+
41+
func TestRenderVendorData_PackagesAndRunCmd(t *testing.T) {
42+
got, err := RenderVendorData([]string{"open-iscsi", "nfs-common"}, []string{"echo hi", "mkdir -p /data"})
43+
if err != nil {
44+
t.Fatalf("RenderVendorData: %v", err)
45+
}
46+
cfg := parseCloudConfig(t, got)
47+
48+
// package_update must be true whenever packages are requested, so the
49+
// index is refreshed before install.
50+
if cfg["package_update"] != true {
51+
t.Errorf("package_update = %v, want true", cfg["package_update"])
52+
}
53+
if pkgs := asStrings(t, cfg["packages"], "packages"); !equal(pkgs, []string{"open-iscsi", "nfs-common"}) {
54+
t.Errorf("packages = %v", pkgs)
55+
}
56+
// runcmd must lead with the qemu-guest-agent enablement (IP discovery),
57+
// then the user's commands in order.
58+
want := []string{
59+
"systemctl enable qemu-guest-agent",
60+
"systemctl start qemu-guest-agent",
61+
"echo hi",
62+
"mkdir -p /data",
63+
}
64+
if cmds := asStrings(t, cfg["runcmd"], "runcmd"); !equal(cmds, want) {
65+
t.Errorf("runcmd = %v, want %v", cmds, want)
66+
}
67+
}
68+
69+
func TestRenderVendorData_NoPackages(t *testing.T) {
70+
got, err := RenderVendorData(nil, []string{"touch /ready"})
71+
if err != nil {
72+
t.Fatalf("RenderVendorData: %v", err)
73+
}
74+
cfg := parseCloudConfig(t, got)
75+
// No packages → no package_update, no packages key (omitempty).
76+
if _, ok := cfg["package_update"]; ok {
77+
t.Errorf("package_update should be omitted when no packages")
78+
}
79+
if _, ok := cfg["packages"]; ok {
80+
t.Errorf("packages should be omitted when empty")
81+
}
82+
want := []string{
83+
"systemctl enable qemu-guest-agent",
84+
"systemctl start qemu-guest-agent",
85+
"touch /ready",
86+
}
87+
if cmds := asStrings(t, cfg["runcmd"], "runcmd"); !equal(cmds, want) {
88+
t.Errorf("runcmd = %v, want %v", cmds, want)
89+
}
90+
}
91+
92+
func TestRenderVendorData_OnlyAgentWhenEmpty(t *testing.T) {
93+
got, err := RenderVendorData(nil, nil)
94+
if err != nil {
95+
t.Fatalf("RenderVendorData: %v", err)
96+
}
97+
cfg := parseCloudConfig(t, got)
98+
want := []string{
99+
"systemctl enable qemu-guest-agent",
100+
"systemctl start qemu-guest-agent",
101+
}
102+
if cmds := asStrings(t, cfg["runcmd"], "runcmd"); !equal(cmds, want) {
103+
t.Errorf("runcmd = %v, want %v", cmds, want)
104+
}
105+
}
106+
107+
// TestRenderVendorData_EscapesSpecialChars proves arbitrary runcmd strings with
108+
// YAML-significant characters round-trip intact — the reason rendering goes
109+
// through yaml.Marshal instead of hand-concatenation.
110+
func TestRenderVendorData_EscapesSpecialChars(t *testing.T) {
111+
tricky := []string{
112+
`sh -c "echo key: value > /etc/x.conf"`, // colon-space + quotes
113+
`printf '%s\n' "a: b"`, // embedded colon
114+
`echo '#not a comment'`, // leading hash inside quotes
115+
}
116+
got, err := RenderVendorData([]string{"pkg-with-dash"}, tricky)
117+
if err != nil {
118+
t.Fatalf("RenderVendorData: %v", err)
119+
}
120+
cfg := parseCloudConfig(t, got)
121+
cmds := asStrings(t, cfg["runcmd"], "runcmd")
122+
// The two agent commands prefix the three tricky ones.
123+
if len(cmds) != 5 {
124+
t.Fatalf("runcmd length = %d, want 5", len(cmds))
125+
}
126+
if !equal(cmds[2:], tricky) {
127+
t.Errorf("tricky commands did not round-trip:\n got %q\n want %q", cmds[2:], tricky)
128+
}
129+
}
130+
131+
func TestVendorSnippetName(t *testing.T) {
132+
if got := VendorSnippetName(1234); got != "openctl-vendor-1234.yaml" {
133+
t.Errorf("VendorSnippetName(1234) = %q", got)
134+
}
135+
// Distinct per VM so contents never collide.
136+
if VendorSnippetName(1) == VendorSnippetName(2) {
137+
t.Errorf("VendorSnippetName must be unique per vmid")
138+
}
139+
}
140+
141+
func equal(a, b []string) bool {
142+
if len(a) != len(b) {
143+
return false
144+
}
145+
for i := range a {
146+
if a[i] != b[i] {
147+
return false
148+
}
149+
}
150+
return true
151+
}

pkg/proxmox/handler/handler.go

Lines changed: 44 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -344,7 +344,8 @@ func (h *Handler) createVMFromTemplate(ctx context.Context, name, node string, s
344344
}
345345
}
346346

347-
// If cloud-init is configured, try to enable qemu-guest-agent via cicustom vendor data
347+
// If cloud-init is configured, attach the cicustom vendor snippet
348+
// (qemu-guest-agent enablement + any requested packages/runcmd).
348349
if spec.CloudInit != nil {
349350
// Use config storage if available, otherwise try "local"
350351
snippetStorage := "local"
@@ -353,15 +354,7 @@ func (h *Handler) createVMFromTemplate(ctx context.Context, name, node string, s
353354
snippetStorage = s
354355
}
355356
}
356-
if err := h.client.EnsureQemuAgentSnippet(ctx, node, snippetStorage); err != nil {
357-
debugf("createVMFromTemplate: failed to ensure snippet: %v", err)
358-
} else {
359-
cicustom := fmt.Sprintf("vendor=%s:snippets/%s", snippetStorage, client.QemuAgentSnippetName)
360-
debugf("createVMFromTemplate: setting cicustom=%s", cicustom)
361-
if err := h.client.ConfigureVM(ctx, node, vmid, map[string]any{"cicustom": cicustom}); err != nil {
362-
debugf("createVMFromTemplate: failed to set cicustom: %v", err)
363-
}
364-
}
357+
h.applyCloudInitVendorSnippet(ctx, node, vmid, snippetStorage, spec.CloudInit)
365358
}
366359

367360
if spec.StartOnCreate {
@@ -376,6 +369,42 @@ func (h *Handler) createVMFromTemplate(ctx context.Context, name, node string, s
376369
}, nil
377370
}
378371

372+
// applyCloudInitVendorSnippet attaches a cicustom vendor snippet to the VM. When
373+
// the VM declares custom packages/runcmd it uploads a per-VM combined snippet
374+
// (qemu-guest-agent enablement folded in, so IP discovery still works);
375+
// otherwise it uses the shared static qemu-agent snippet — the pre-existing,
376+
// metal-validated path, byte-identical.
377+
//
378+
// Every failure is non-fatal and surfaced via debug logs: a storage that can't
379+
// hold snippets (e.g. LVM) just means no agent/packages, not a failed create —
380+
// matching the prior behavior.
381+
func (h *Handler) applyCloudInitVendorSnippet(ctx context.Context, node string, vmid int, snippetStorage string, ci *resources.CloudInitSpec) {
382+
var snippetName string
383+
if ci != nil && (len(ci.Packages) > 0 || len(ci.RunCmd) > 0) {
384+
content, err := client.RenderVendorData(ci.Packages, ci.RunCmd)
385+
if err != nil {
386+
debugf("applyCloudInitVendorSnippet: render vendor data: %v", err)
387+
return
388+
}
389+
snippetName = client.VendorSnippetName(vmid)
390+
if err := h.client.UploadSnippet(ctx, node, snippetStorage, snippetName, content); err != nil {
391+
debugf("applyCloudInitVendorSnippet: upload vendor snippet: %v", err)
392+
return
393+
}
394+
} else {
395+
if err := h.client.EnsureQemuAgentSnippet(ctx, node, snippetStorage); err != nil {
396+
debugf("applyCloudInitVendorSnippet: ensure agent snippet: %v", err)
397+
return
398+
}
399+
snippetName = client.QemuAgentSnippetName
400+
}
401+
cicustom := fmt.Sprintf("vendor=%s:snippets/%s", snippetStorage, snippetName)
402+
debugf("applyCloudInitVendorSnippet: setting cicustom=%s", cicustom)
403+
if err := h.client.ConfigureVM(ctx, node, vmid, map[string]any{"cicustom": cicustom}); err != nil {
404+
debugf("applyCloudInitVendorSnippet: set cicustom: %v", err)
405+
}
406+
}
407+
379408
func primaryDiskStorage(spec *resources.VMSpec) string {
380409
if spec == nil {
381410
return ""
@@ -475,20 +504,11 @@ func (h *Handler) createVMFromCloudImage(ctx context.Context, name, node string,
475504
}
476505
}
477506

478-
// Enable qemu-guest-agent via cicustom vendor data (for IP detection)
479-
// Using vendor= instead of user= so it merges with cloud-init config instead of replacing it
480-
snippetStorage := spec.CloudImage.Storage
481-
if err := h.client.EnsureQemuAgentSnippet(ctx, node, snippetStorage); err != nil {
482-
// Try to continue - storage might not support snippets
483-
debugf("createVMFromCloudImage: failed to ensure snippet: %v", err)
484-
} else {
485-
// Add cicustom vendor data to VM config
486-
cicustom := fmt.Sprintf("vendor=%s:snippets/%s", snippetStorage, client.QemuAgentSnippetName)
487-
debugf("createVMFromCloudImage: setting cicustom=%s", cicustom)
488-
if err := h.client.ConfigureVM(ctx, node, vmid, map[string]any{"cicustom": cicustom}); err != nil {
489-
debugf("createVMFromCloudImage: failed to set cicustom: %v", err)
490-
}
491-
}
507+
// Attach the cicustom vendor snippet (qemu-guest-agent enablement for IP
508+
// detection, plus any requested packages/runcmd). Using vendor= (not
509+
// user=) so it merges with Proxmox's generated cloud-init user-data
510+
// instead of replacing it.
511+
h.applyCloudInitVendorSnippet(ctx, node, vmid, spec.CloudImage.Storage, spec.CloudInit)
492512

493513
// Regenerate cloud-init with new settings (error ignored - non-fatal)
494514
if spec.CloudInit != nil {

0 commit comments

Comments
 (0)