Merge pull request #44 from stacklok/feat/configurable-tmp-size

Add configurable /tmp tmpfs size for guest VMs

Author: JAORMX

guest/boot/boot.go

@@ -45,7 +45,7 @@ func Run(logger *slog.Logger, opts ...Option) (shutdown func(), err error) {
 
 	// 1. Essential mounts — /proc is needed before netlink can work.
 	logger.Info("mounting essential filesystems")
-	if err := mount.Essential(logger); err != nil {
+	if err := mount.Essential(logger, cfg.tmpSizeMiB); err != nil {
 		return nil, fmt.Errorf("essential mounts: %w", err)
 	}
 

guest/boot/options.go

@@ -33,6 +33,7 @@ type config struct {
 	lockdownRoot        bool
 	sshAgentForwarding  bool
 	seccomp             bool
+	tmpSizeMiB          uint32
 }
 
 func defaultConfig() *config {
@@ -116,6 +117,17 @@ func WithSSHAgentForwarding(enabled bool) Option {
 	return optionFunc(func(c *config) { c.sshAgentForwarding = enabled })
 }
 
+// WithTmpSize sets the size of the /tmp tmpfs in MiB. Defaults to 256 MiB when
+// 0 or not set. The value is read from /etc/propolis-vm.json when provided by
+// the host via [github.com/stacklok/propolis.WithTmpSize].
+func WithTmpSize(mib uint32) Option {
+	return optionFunc(func(c *config) {
+		if mib > 0 {
+			c.tmpSizeMiB = mib
+		}
+	})
+}
+
 // WithSeccomp controls whether a seccomp BPF blocklist filter is
 // applied as the last step of the boot sequence. When enabled, the
 // filter blocks dangerous syscalls (io_uring, ptrace, bpf, mount,

guest/mount/export_test.go

@@ -0,0 +1,9 @@
+// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+//go:build linux
+
+package mount
+
+// EssentialMountsForTest exposes essentialMounts for testing.
+var EssentialMountsForTest = essentialMounts

guest/mount/mount.go

@@ -14,6 +14,8 @@ import (
 	"time"
 )
 
+const defaultTmpSizeMiB = 256
+
 type mountEntry struct {
 	source string
 	target string
@@ -22,16 +24,26 @@ type mountEntry struct {
 	data   string
 }
 
-// Essential mounts the core filesystems required for a minimal Linux userspace.
-func Essential(logger *slog.Logger) error {
-	mounts := []mountEntry{
+// essentialMounts returns the mount table for Essential, applying the default
+// tmp size when tmpSizeMiB is zero.
+func essentialMounts(tmpSizeMiB uint32) []mountEntry {
+	if tmpSizeMiB == 0 {
+		tmpSizeMiB = defaultTmpSizeMiB
+	}
+	return []mountEntry{
 		{"proc", "/proc", "proc", syscall.MS_NOSUID | syscall.MS_NODEV | syscall.MS_NOEXEC, ""},
 		{"sysfs", "/sys", "sysfs", syscall.MS_NOSUID | syscall.MS_NODEV | syscall.MS_NOEXEC, ""},
 		{"devtmpfs", "/dev", "devtmpfs", syscall.MS_NOSUID | syscall.MS_NOEXEC, ""},
 		{"devpts", "/dev/pts", "devpts", syscall.MS_NOSUID | syscall.MS_NOEXEC, "newinstance,ptmxmode=0666,mode=0620,gid=5"},
-		{"tmpfs", "/tmp", "tmpfs", syscall.MS_NOSUID | syscall.MS_NODEV, "size=256m"},
+		{"tmpfs", "/tmp", "tmpfs", syscall.MS_NOSUID | syscall.MS_NODEV, fmt.Sprintf("size=%dm", tmpSizeMiB)},
 		{"tmpfs", "/run", "tmpfs", syscall.MS_NOSUID | syscall.MS_NODEV | syscall.MS_NOEXEC, "size=64m"},
 	}
+}
+
+// Essential mounts the core filesystems required for a minimal Linux userspace.
+// tmpSizeMiB sets the size of the /tmp tmpfs in MiB; 0 uses the default (256 MiB).
+func Essential(logger *slog.Logger, tmpSizeMiB uint32) error {
+	mounts := essentialMounts(tmpSizeMiB)
 
 	for _, m := range mounts {
 		if err := os.MkdirAll(m.target, 0o755); err != nil {

guest/mount/mount_test.go

@@ -18,7 +18,7 @@ func TestEssentialRequiresRoot(t *testing.T) {
 	if os.Getuid() == 0 {
 		t.Skip("test must run as non-root")
 	}
-	err := Essential(slog.Default())
+	err := Essential(slog.Default(), 0)
 	assert.Error(t, err)
 }
 
@@ -31,20 +31,30 @@ func TestWorkspaceReturnsErrorForInvalidMount(t *testing.T) {
 	assert.Error(t, err)
 }
 
-func TestEssentialMountPoints(t *testing.T) {
+func TestEssentialMounts(t *testing.T) {
 	t.Parallel()
 
-	// Verify the mount table contains the expected entries.
-	expected := []string{"/proc", "/sys", "/dev", "/dev/pts", "/tmp", "/run"}
-	mounts := []mountEntry{
-		{"proc", "/proc", "proc", 0, ""},
-		{"sysfs", "/sys", "sysfs", 0, ""},
-		{"devtmpfs", "/dev", "devtmpfs", 0, ""},
-		{"devpts", "/dev/pts", "devpts", 0, "newinstance,ptmxmode=0666,mode=0620,gid=5"},
-		{"tmpfs", "/tmp", "tmpfs", 0, "size=256m"},
-		{"tmpfs", "/run", "tmpfs", 0, "size=64m"},
-	}
-	for i, m := range mounts {
-		assert.Equal(t, expected[i], m.target)
-	}
+	t.Run("default targets", func(t *testing.T) {
+		t.Parallel()
+		mounts := EssentialMountsForTest(0)
+		expected := []string{"/proc", "/sys", "/dev", "/dev/pts", "/tmp", "/run"}
+		targets := make([]string, len(mounts))
+		for i, m := range mounts {
+			targets[i] = m.target
+		}
+		assert.Equal(t, expected, targets)
+	})
+
+	t.Run("zero uses default tmp size", func(t *testing.T) {
+		t.Parallel()
+		mounts := EssentialMountsForTest(0)
+		// /tmp is the 5th entry.
+		assert.Equal(t, "size=256m", mounts[4].data)
+	})
+
+	t.Run("custom tmp size flows through", func(t *testing.T) {
+		t.Parallel()
+		mounts := EssentialMountsForTest(512)
+		assert.Equal(t, "size=512m", mounts[4].data)
+	})
 }

guest/vmconfig/doc.go

@@ -0,0 +1,7 @@
+// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package vmconfig reads the VM configuration file written by the host into
+// the rootfs before boot. Guest init binaries use this to apply host-side
+// configuration (e.g. /tmp size) before the SSH server starts.
+package vmconfig

guest/vmconfig/export_test.go

@@ -0,0 +1,7 @@
+// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package vmconfig
+
+// ReadFromForTest exposes readFrom for testing.
+var ReadFromForTest = readFrom

guest/vmconfig/vmconfig.go

@@ -0,0 +1,45 @@
+// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package vmconfig
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+)
+
+// GuestPath is the guest path where the host writes the VM config.
+const GuestPath = "/etc/propolis-vm.json"
+
+// Config holds settings written by the host and read by the guest init.
+// Zero values mean "use the built-in default" for each field.
+type Config struct {
+	// TmpSizeMiB is the size of the /tmp tmpfs in MiB. Zero means use the
+	// mount package default (256 MiB).
+	TmpSizeMiB uint32 `json:"tmp_size_mib,omitempty"`
+}
+
+// Read loads the VM config from /etc/propolis-vm.json.
+// Returns a zero-value Config (all defaults) if the file does not exist,
+// ensuring backward compatibility with hosts that do not write the file.
+func Read() (Config, error) {
+	return readFrom(GuestPath)
+}
+
+// readFrom loads the VM config from the given path.
+func readFrom(path string) (Config, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return Config{}, nil
+		}
+		return Config{}, fmt.Errorf("reading vm config: %w", err)
+	}
+	var cfg Config
+	if err := json.Unmarshal(data, &cfg); err != nil {
+		return Config{}, fmt.Errorf("parsing vm config: %w", err)
+	}
+	return cfg, nil
+}

guest/vmconfig/vmconfig_test.go

@@ -0,0 +1,68 @@
+// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package vmconfig
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestReadFrom(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name    string
+		content *string // nil = file does not exist
+		want    Config
+		wantErr bool
+	}{
+		{
+			name:    "file does not exist",
+			content: nil,
+			want:    Config{},
+		},
+		{
+			name:    "valid JSON with TmpSizeMiB",
+			content: strPtr(`{"tmp_size_mib":512}`),
+			want:    Config{TmpSizeMiB: 512},
+		},
+		{
+			name:    "empty JSON object",
+			content: strPtr(`{}`),
+			want:    Config{},
+		},
+		{
+			name:    "malformed JSON",
+			content: strPtr(`{not json`),
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			dir := t.TempDir()
+			path := filepath.Join(dir, "vm.json")
+
+			if tt.content != nil {
+				require.NoError(t, os.WriteFile(path, []byte(*tt.content), 0o644))
+			}
+
+			got, err := ReadFromForTest(path)
+			if tt.wantErr {
+				assert.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}
+
+func strPtr(s string) *string { return &s }

hooks/hooks.go

@@ -4,6 +4,7 @@
 package hooks
 
 import (
+	"encoding/json"
 	"fmt"
 	"log/slog"
 	"os"
@@ -12,11 +13,25 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/stacklok/propolis/guest/vmconfig"
 	"github.com/stacklok/propolis/image"
 	"github.com/stacklok/propolis/internal/pathutil"
 	"github.com/stacklok/propolis/internal/xattr"
 )
 
+// InjectVMConfig returns a RootFSHook that writes the given VM config as JSON
+// to /etc/propolis-vm.json inside the rootfs. The guest init reads this file
+// to configure mounts before the SSH server starts.
+func InjectVMConfig(cfg vmconfig.Config) func(string, *image.OCIConfig) error {
+	return func(rootfsPath string, _ *image.OCIConfig) error {
+		data, err := json.Marshal(cfg)
+		if err != nil {
+			return fmt.Errorf("marshal vm config: %w", err)
+		}
+		return InjectFile(vmconfig.GuestPath, data, 0o644)(rootfsPath, nil)
+	}
+}
+
 // validEnvKey matches POSIX-compliant environment variable names.
 var validEnvKey = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)