diff --git a/packages/orchestrator/internal/template/build/core/filesystem/ext4.go b/packages/orchestrator/internal/template/build/core/filesystem/ext4.go index d691c908e7..6022ed7ee1 100644 --- a/packages/orchestrator/internal/template/build/core/filesystem/ext4.go +++ b/packages/orchestrator/internal/template/build/core/filesystem/ext4.go @@ -25,7 +25,7 @@ var tracer = otel.Tracer("github.com/e2b-dev/infra/packages/orchestrator/interna const ( // creates an inode for every bytes-per-inode byte of space on the disk inodesRatio = int64(4096) - // Percentage of reserved blocks in the filesystem + // reservedBlocksPercentage is 0 because reserved blocks are set post-creation via tune2fs -r after the final resize. reservedBlocksPercentage = int64(0) ToMBShift = 20 @@ -42,8 +42,13 @@ func Make(ctx context.Context, rootfsPath string, sizeMb int64, blockSize int64) cmd := exec.CommandContext(ctx, "mkfs.ext4", // Matches the final ext4 features used by tar2ext4 tool - // But enables resize_inode, sparse_super (default, required for resize_inode), has_journal (default), metadata_csum (default) - "-O", `^dir_index,^64bit,^dir_nlink,ext_attr,sparse_super2,filetype,extent,flex_bg,large_file,huge_file,extra_isize`, + // But enables resize_inode, sparse_super (default, required for resize_inode), has_journal (default), metadata_csum (default). + // orphan_file is disabled (added as default in e2fsprogs >= 1.47.0) to ensure guest e2fsprogs tools + // (tune2fs, resize2fs, e2fsck) from older images (e.g. Ubuntu 22.04, Debian 11) can write to + // the filesystem. Without this, any write operation from the guest fails with "unsupported + // read-only feature(s)" when the host e2fsprogs is newer than the guest's. + // See https://e2fsprogs.sourceforge.net/e2fsprogs-release.html#1.47.0 + "-O", `^dir_index,^64bit,^dir_nlink,^orphan_file,ext_attr,sparse_super2,filetype,extent,flex_bg,large_file,huge_file,extra_isize`, "-b", strconv.FormatInt(blockSize, 10), "-m", strconv.FormatInt(reservedBlocksPercentage, 10), "-i", strconv.FormatInt(inodesRatio, 10), @@ -308,6 +313,33 @@ func MountOverlayFS(ctx context.Context, layers []string, mountPoint string) err return nil } +// SetReservedBlocksOnHost sets the number of reserved filesystem blocks based on the desired reserved space in MB. +// Reserved blocks are only usable by root (uid 0). +func SetReservedBlocksOnHost(ctx context.Context, rootfsPath string, reservedSpaceMB int64, blockSize int64) error { + if reservedSpaceMB <= 0 { + return nil + } + + ctx, span := tracer.Start(ctx, "set-reserved-blocks") + defer span.End() + + blocks := (reservedSpaceMB << ToMBShift) / blockSize + + cmd := exec.CommandContext(ctx, "tune2fs", "-r", strconv.FormatInt(blocks, 10), rootfsPath) + + stdoutWriter := telemetry.NewEventWriter(ctx, "stdout") + cmd.Stdout = stdoutWriter + + stderrWriter := telemetry.NewEventWriter(ctx, "stderr") + cmd.Stderr = stderrWriter + + if err := cmd.Run(); err != nil { + return fmt.Errorf("error setting reserved blocks: %w", err) + } + + return nil +} + func LogMetadata(ctx context.Context, rootfsPath string, extraFields ...zap.Field) { cmd := exec.CommandContext(ctx, "tune2fs", "-l", rootfsPath) output, err := cmd.CombinedOutput() diff --git a/packages/orchestrator/internal/template/build/core/filesystem/ext4_test.go b/packages/orchestrator/internal/template/build/core/filesystem/ext4_test.go new file mode 100644 index 0000000000..b231fa6a8f --- /dev/null +++ b/packages/orchestrator/internal/template/build/core/filesystem/ext4_test.go @@ -0,0 +1,90 @@ +package filesystem + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseFreeBlocks(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input string + expected int64 + wantErr bool + }{ + { + name: "standard debugfs output", + input: "Block count: 131072\nFree blocks: 120000\nFirst block: 0\n", + expected: 120000, + }, + { + name: "large block count", + input: "Free blocks: 999999999\n", + expected: 999999999, + }, + { + name: "missing free blocks", + input: "Block count: 131072\n", + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + result, err := parseFreeBlocks(tc.input) + if tc.wantErr { + assert.Error(t, err) + } else { + require.NoError(t, err) + assert.Equal(t, tc.expected, result) + } + }) + } +} + +func TestParseReservedBlocks(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input string + expected int64 + wantErr bool + }{ + { + name: "standard debugfs output", + input: "Block count: 131072\nReserved block count: 6553\nFree blocks: 120000\n", + expected: 6553, + }, + { + name: "zero reserved blocks", + input: "Reserved block count: 0\n", + expected: 0, + }, + { + name: "missing reserved blocks", + input: "Block count: 131072\nFree blocks: 120000\n", + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + result, err := parseReservedBlocks(tc.input) + if tc.wantErr { + assert.Error(t, err) + } else { + require.NoError(t, err) + assert.Equal(t, tc.expected, result) + } + }) + } +} diff --git a/packages/orchestrator/internal/template/build/phases/base/builder.go b/packages/orchestrator/internal/template/build/phases/base/builder.go index 61c34dee8b..8faebd75fb 100644 --- a/packages/orchestrator/internal/template/build/phases/base/builder.go +++ b/packages/orchestrator/internal/template/build/phases/base/builder.go @@ -252,6 +252,13 @@ func (bb *BaseBuilder) buildLayerFromOCI( return metadata.Template{}, fmt.Errorf("error enlarging disk after provisioning: %w", err) } + if reservedDiskSpaceMB := int64(bb.featureFlags.IntFlag(ctx, featureflags.BuildReservedDiskSpaceMB)); reservedDiskSpaceMB > 0 { + err = filesystem.SetReservedBlocksOnHost(ctx, rootfsPath, reservedDiskSpaceMB, bb.Config.RootfsBlockSize()) + if err != nil { + return metadata.Template{}, fmt.Errorf("error setting reserved disk space: %w", err) + } + } + // Create sandbox for building template userLogger.Debug(ctx, "Creating base sandbox template layer") diff --git a/packages/orchestrator/internal/template/build/phases/finalize/builder.go b/packages/orchestrator/internal/template/build/phases/finalize/builder.go index a90ff58d99..fcb89e02f8 100644 --- a/packages/orchestrator/internal/template/build/phases/finalize/builder.go +++ b/packages/orchestrator/internal/template/build/phases/finalize/builder.go @@ -219,6 +219,16 @@ func (ppb *PostProcessingBuilder) postProcessingFn(userLogger logger.Logger) lay return } + // Set reserved disk space for the guest OS before syncing + if reservedDiskSpaceMB := int64(ppb.featureFlags.IntFlag(ctx, featureflags.BuildReservedDiskSpaceMB)); reservedDiskSpaceMB > 0 { + err := sandboxtools.SetReservedBlocksInGuest(ctx, ppb.proxy, userLogger, sbx.Runtime.SandboxID, reservedDiskSpaceMB, ppb.Config.RootfsBlockSize()) + if err != nil { + e = fmt.Errorf("error setting reserved disk space: %w", err) + + return + } + } + // Ensure all changes are synchronized to disk so the sandbox can be restarted err := sandboxtools.SyncChangesToDisk( ctx, diff --git a/packages/orchestrator/internal/template/build/sandboxtools/command.go b/packages/orchestrator/internal/template/build/sandboxtools/command.go index 47c1a1eca1..1a23e8f111 100644 --- a/packages/orchestrator/internal/template/build/sandboxtools/command.go +++ b/packages/orchestrator/internal/template/build/sandboxtools/command.go @@ -13,11 +13,13 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/trace" + "go.uber.org/zap" "go.uber.org/zap/zapcore" "github.com/e2b-dev/infra/packages/orchestrator/internal/proxy" "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox" "github.com/e2b-dev/infra/packages/orchestrator/internal/template/build/core/rootfs" + "github.com/e2b-dev/infra/packages/orchestrator/internal/template/constants" "github.com/e2b-dev/infra/packages/orchestrator/internal/template/metadata" "github.com/e2b-dev/infra/packages/shared/pkg/grpc" "github.com/e2b-dev/infra/packages/shared/pkg/grpc/envd/process" @@ -232,6 +234,38 @@ func logStream(ctx context.Context, logger logger.Logger, lvl zapcore.Level, id } } +// SetReservedBlocksInGuest sets the number of reserved filesystem blocks inside the sandbox. +// Reserved blocks are only usable by root (uid 0), protecting the guest OS from disk-full conditions. +// Requires e2fsprogs (tune2fs) installed in the guest image (standard on Debian-based images). +func SetReservedBlocksInGuest( + ctx context.Context, + proxy *proxy.SandboxProxy, + logger logger.Logger, + sandboxID string, + reservedSpaceMB int64, + blockSize int64, +) error { + if reservedSpaceMB <= 0 { + return nil + } + + blocks := (reservedSpaceMB << constants.ToMBShift) / blockSize + tuneCmd := fmt.Sprintf("tune2fs -r %d /dev/vda", blocks) + + return RunCommandWithLogger( + ctx, + proxy, + logger, + zap.DebugLevel, + "set-reserved-disk-space", + sandboxID, + tuneCmd, + metadata.Context{ + User: "root", + }, + ) +} + // syncChangesToDisk synchronizes filesystem changes to the filesystem // This is useful to ensure that all changes made in the sandbox are written to disk // to be able to re-create the sandbox without resume. diff --git a/packages/shared/pkg/featureflags/flags.go b/packages/shared/pkg/featureflags/flags.go index a15142ee1a..e581359d95 100644 --- a/packages/shared/pkg/featureflags/flags.go +++ b/packages/shared/pkg/featureflags/flags.go @@ -177,6 +177,10 @@ var ( // BuildBaseRootfsSizeLimitMB is the maximum size of the base rootfs filesystem created from the OCI image, in MB. BuildBaseRootfsSizeLimitMB = newIntFlag("build-base-rootfs-size-limit-mb", 25000) + // BuildReservedDiskSpaceMB is the amount of disk space in MB reserved for root on the guest filesystem. + // Reserved blocks are only usable by root (uid 0), protecting the guest OS from disk-full conditions. + BuildReservedDiskSpaceMB = newIntFlag("build-reserved-disk-space-mb", 0) + // MaxConcurrentSnapshotUpserts limits concurrent UpsertSnapshot calls (pause + snapshot template paths). // 0 or negative disables throttling (unlimited concurrency). MaxConcurrentSnapshotUpserts = newIntFlag("max-concurrent-snapshot-upserts", 0)