Filesystem-only snapshots for auto-pause (#3055)

Summary

Adds a per-sandbox autoPauseMemory create option (default true). When
false, a sandbox that auto-pauses on timeout takes a filesystem-only
snapshot (no memory) instead of a full memory snapshot, so it cold-boots
from disk on resume. Builds on the filesystem-only snapshot machinery
from #3027 — this PR just lets the timeout auto-pause path opt into it,
where
previously only an explicit pause(memory=false) could.

How it works

The FilesystemOnly flag was already plumbed end-to-end through the pause
path (RemoveOpts → pauseSandbox → snapshot → Pause RPC); the only
missing piece was a per-sandbox policy for the evictor to act on.

1. Create — autoPauseMemory=false → Sandbox.AutoPauseFilesystemOnly=true
(runtime state, persisted to Redis).
2. Evict — when the evictor auto-pauses on timeout, it threads
opts.FilesystemOnly = sbx.AutoPauseFilesystemOnly.
3. Persist — the policy is stamped onto the snapshot's
PausedSandboxConfig, and...
4. Restore — re-read on resume (covers explicit /resume, /connect, and
auto-resume — they share buildResumeSandboxData), so the policy survives
a pause/resume cycle.
5. Re-sync — also carried through
SandboxConfig.auto_pause_filesystem_only in the orchestrator proto,
mirroring auto_pause, so an API restart that rebuilds from the
orchestrator's sandbox list doesn't lose it.

The orchestrator is unchanged — the existing IsFilesystemOnly() dispatch
already routes any resume of an fs-only snapshot to RebootSandbox.

Constraints (rejected at create, 400)

- autoPauseMemory=false requires autoPause=true (it only governs the
timeout auto-pause; otherwise a no-op).
- autoPauseMemory=false is incompatible with autoResume — a
filesystem-only snapshot can't be auto-resumed by arbitrary traffic
(it'd cold-boot, losing processes/connections) and must be resumed
explicitly.

Design notes

- Two distinct fields. AutoPauseFilesystemOnly (the policy for the next
auto-pause) is kept separate from FilesystemOnly (the kind of this
snapshot), so an explicit pause(memory=true) on such a sandbox produces
a memory snapshot yet still restores the fs-only policy on the next
cycle.
- Resume inherits the policy. Unlike auto_pause (overridable via the
resume body, now deprecated), the auto-pause kind is always inherited
from the snapshot — changing it means creating a new sandbox.
Deliberate, to avoid extending a deprecated path.
- Backward-compatible: default true, omitempty on both the JSON config
and the proto, so existing sandboxes/snapshots decode to memory
auto-pause.

Testing

- Unit — evictor wiring (kill/pause × snapshot kind);
PausedSandboxConfig serialization round-trip + legacy-row default; proto
re-sync round-trip via GetSandboxes.
- Integration — create → timeout auto-pause → resume cold-boots (rootfs
marker survives, fresh boot id, correct default user), and the policy
persists across a second pause/resume cycle; both create-time guards
return 400.

Usage

```
# auto-pause this sandbox as a filesystem-only snapshot on timeout
POST /sandboxes { "templateID": "...", "autoPause": true, "autoPauseMemory": false }
```

---------

Signed-off-by: Babis Chalios <babis.chalios@e2b.dev>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>

Author: bchalios

packages/api/internal/api/api.gen.go

@@ -142,6 +142,9 @@ func (a *APIStore) PostSandboxes(c *gin.Context) {
 	)
 
 	autoPause := sharedUtils.DerefOrDefault(body.AutoPause, sandbox.AutoPauseDefault)
+	// autoPauseMemory defaults to true (full memory snapshot). When false, a
+	// timeout auto-pause takes a filesystem-only snapshot (cold-boots on resume).
+	autoPauseFilesystemOnly := !sharedUtils.DerefOrDefault(body.AutoPauseMemory, true)
 	envVars := sharedUtils.DerefOrDefault(body.EnvVars, nil)
 	mcp := sharedUtils.DerefOrDefault(body.Mcp, nil)
 	metadata := sharedUtils.DerefOrDefault(body.Metadata, nil)
@@ -164,6 +167,23 @@ func (a *APIStore) PostSandboxes(c *gin.Context) {
 		autoResume.Timeout = calculateTimeoutSeconds(timeout, minAutoResumeTimeout, teamInfo)
 	}
 
+	// autoPauseMemory only controls the snapshot kind of a timeout auto-pause, so
+	// it is meaningless without autoPause; reject it rather than silently storing
+	// a no-op policy.
+	if autoPauseFilesystemOnly && !autoPause {
+		a.sendAPIStoreError(c, http.StatusBadRequest, "autoPauseMemory=false only applies when autoPause is true.")
+
+		return
+	}
+
+	// A filesystem-only auto-pause produces a snapshot that traffic cannot
+	// auto-resume (it must be resumed explicitly), so the two are incompatible.
+	if autoPauseFilesystemOnly && autoResume != nil && autoResume.Policy == types.SandboxAutoResumeAny {
+		a.sendAPIStoreError(c, http.StatusBadRequest, "autoPauseMemory=false (filesystem-only auto-pause) cannot be combined with autoResume: a filesystem-only snapshot cannot be auto-resumed by traffic and must be resumed explicitly.")
+
+		return
+	}
+
 	var envdAccessToken *string = nil
 	if body.Secure != nil && *body.Secure == true {
 		accessToken, tokenErr := a.getEnvdAccessToken(build.EnvdVersion, sandboxID)
@@ -268,18 +288,19 @@ func (a *APIStore) PostSandboxes(c *gin.Context) {
 		// The data can't be influenced by action on the same sandbox as other operations,
 		// so it's safe to reuse the data
 		return apiorch.SandboxMetadata{
-			Metadata:            metadata,
-			EnvVars:             envVars,
-			Build:               *build,
-			AllowInternetAccess: allowInternetAccess,
-			Network:             network,
-			Alias:               alias,
-			TemplateID:          env.TemplateID,
-			BaseTemplateID:      env.TemplateID,
-			AutoPause:           autoPause,
-			AutoResume:          autoResume,
-			VolumeMounts:        sbxVolumeMounts,
-			EnvdAccessToken:     envdAccessToken,
+			Metadata:                metadata,
+			EnvVars:                 envVars,
+			Build:                   *build,
+			AllowInternetAccess:     allowInternetAccess,
+			Network:                 network,
+			Alias:                   alias,
+			TemplateID:              env.TemplateID,
+			BaseTemplateID:          env.TemplateID,
+			AutoPause:               autoPause,
+			AutoPauseFilesystemOnly: autoPauseFilesystemOnly,
+			AutoResume:              autoResume,
+			VolumeMounts:            sbxVolumeMounts,
+			EnvdAccessToken:         envdAccessToken,
 		}, nil
 	}
 

packages/api/internal/handlers/sandbox_create.go

@@ -227,25 +227,32 @@ func (a *APIStore) buildResumeSandboxData(sandboxID string, autoPauseOverride *b
 		var network *types.SandboxNetworkConfig
 		var autoResume *types.SandboxAutoResumeConfig
 		var volumes []*types.SandboxVolumeMountConfig
+		// Unlike auto_pause (which resume can override via the request body), the
+		// auto-pause snapshot kind is intentionally always inherited from the
+		// snapshot: there is no resume-time override for it. Changing the kind
+		// requires creating a new sandbox with the desired autoPauseMemory.
+		var autoPauseFilesystemOnly bool
 		if snap.Config != nil {
 			network = snap.Config.Network
 			autoResume = snap.Config.AutoResume
 			volumes = snap.Config.VolumeMounts
+			autoPauseFilesystemOnly = snap.Config.AutoPauseFilesystemOnly
 		}
 
 		return orchestrator.SandboxMetadata{
-			Metadata:            snap.Metadata,
-			Build:               build,
-			AllowInternetAccess: snap.AllowInternetAccess,
-			Network:             network,
-			Alias:               alias,
-			TemplateID:          snap.EnvID,
-			BaseTemplateID:      snap.BaseEnvID,
-			AutoPause:           autoPause,
-			AutoResume:          autoResume,
-			VolumeMounts:        convertDatabaseMountsToOrchestratorMounts(volumes),
-			EnvdAccessToken:     envdAccessToken,
-			NodeID:              &nodeID,
+			Metadata:                snap.Metadata,
+			Build:                   build,
+			AllowInternetAccess:     snap.AllowInternetAccess,
+			Network:                 network,
+			Alias:                   alias,
+			TemplateID:              snap.EnvID,
+			BaseTemplateID:          snap.BaseEnvID,
+			AutoPause:               autoPause,
+			AutoPauseFilesystemOnly: autoPauseFilesystemOnly,
+			AutoResume:              autoResume,
+			VolumeMounts:            convertDatabaseMountsToOrchestratorMounts(volumes),
+			EnvdAccessToken:         envdAccessToken,
+			NodeID:                  &nodeID,
 		}, nil
 	}
 }

packages/api/internal/handlers/sandbox_resume.go

@@ -47,10 +47,13 @@ type SandboxMetadata struct {
 	TemplateID          string
 	BaseTemplateID      string
 	AutoPause           bool
-	AutoResume          *types.SandboxAutoResumeConfig
-	VolumeMounts        []*orchestrator.SandboxVolumeMount
-	EnvdAccessToken     *string
-	NodeID              *string
+	// AutoPauseFilesystemOnly makes a timeout auto-pause take a filesystem-only
+	// snapshot instead of a full memory one. Only meaningful when AutoPause.
+	AutoPauseFilesystemOnly bool
+	AutoResume              *types.SandboxAutoResumeConfig
+	VolumeMounts            []*orchestrator.SandboxVolumeMount
+	EnvdAccessToken         *string
+	NodeID                  *string
 }
 
 // buildEgressConfig constructs the orchestrator egress configuration from
@@ -268,30 +271,31 @@ func (o *Orchestrator) CreateSandbox(
 
 	sbxRequest := &orchestrator.SandboxCreateRequest{
 		Sandbox: &orchestrator.SandboxConfig{
-			BaseTemplateId:      sbxData.BaseTemplateID,
-			TemplateId:          sbxData.TemplateID,
-			Alias:               &sbxData.Alias,
-			TeamId:              team.ID.String(),
-			BuildId:             sbxData.Build.ID.String(),
-			SandboxId:           sandboxID,
-			ExecutionId:         executionID,
-			KernelVersion:       sbxData.Build.KernelVersion,
-			FirecrackerVersion:  sbxData.Build.FirecrackerVersion,
-			EnvdVersion:         *sbxData.Build.EnvdVersion,
-			Metadata:            sbxData.Metadata,
-			EnvVars:             sbxData.EnvVars,
-			EnvdAccessToken:     sbxData.EnvdAccessToken,
-			MaxSandboxLength:    team.Limits.MaxLengthHours,
-			HugePages:           hasHugePages,
-			RamMb:               sbxData.Build.RamMb,
-			Vcpu:                sbxData.Build.Vcpu,
-			Snapshot:            isResume,
-			AutoPause:           sbxData.AutoPause,
-			AutoResume:          orchAutoResume,
-			AllowInternetAccess: sbxData.AllowInternetAccess,
-			Network:             sbxNetwork,
-			TotalDiskSizeMb:     ut.FromPtr(sbxData.Build.TotalDiskSizeMb),
-			VolumeMounts:        sbxData.VolumeMounts,
+			BaseTemplateId:          sbxData.BaseTemplateID,
+			TemplateId:              sbxData.TemplateID,
+			Alias:                   &sbxData.Alias,
+			TeamId:                  team.ID.String(),
+			BuildId:                 sbxData.Build.ID.String(),
+			SandboxId:               sandboxID,
+			ExecutionId:             executionID,
+			KernelVersion:           sbxData.Build.KernelVersion,
+			FirecrackerVersion:      sbxData.Build.FirecrackerVersion,
+			EnvdVersion:             *sbxData.Build.EnvdVersion,
+			Metadata:                sbxData.Metadata,
+			EnvVars:                 sbxData.EnvVars,
+			EnvdAccessToken:         sbxData.EnvdAccessToken,
+			MaxSandboxLength:        team.Limits.MaxLengthHours,
+			HugePages:               hasHugePages,
+			RamMb:                   sbxData.Build.RamMb,
+			Vcpu:                    sbxData.Build.Vcpu,
+			Snapshot:                isResume,
+			AutoPause:               sbxData.AutoPause,
+			AutoPauseFilesystemOnly: sbxData.AutoPauseFilesystemOnly,
+			AutoResume:              orchAutoResume,
+			AllowInternetAccess:     sbxData.AllowInternetAccess,
+			Network:                 sbxNetwork,
+			TotalDiskSizeMb:         ut.FromPtr(sbxData.Build.TotalDiskSizeMb),
+			VolumeMounts:            sbxData.VolumeMounts,
 		},
 		StartTime: timestamppb.New(startTime),
 		EndTime:   timestamppb.New(endTime),
@@ -360,6 +364,7 @@ func (o *Orchestrator) CreateSandbox(
 		node.ID,
 		node.ClusterID,
 		sbxData.AutoPause,
+		sbxData.AutoPauseFilesystemOnly,
 		sbxData.AutoResume,
 		sbxData.EnvdAccessToken,
 		sbxData.AllowInternetAccess,

packages/api/internal/orchestrator/create_instance.go

@@ -156,8 +156,13 @@ func (e *Evictor) evictSandbox(ctx context.Context, sbx sandbox.Sandbox) {
 	}
 
 	opts := sandbox.RemoveOpts{Action: action, Eviction: true}
-	if action == sandbox.StateActionKill {
+	switch action {
+	case sandbox.StateActionKill:
 		opts.Reason = sandbox.KillReasonTimeout
+	case sandbox.StateActionPause:
+		// Honor the sandbox's auto-pause snapshot kind: filesystem-only drops
+		// memory (cold-boots on resume); otherwise a full memory snapshot.
+		opts.FilesystemOnly = sbx.AutoPauseFilesystemOnly
 	}
 
 	if err := e.removeSandbox(context.WithoutCancel(ctx), sbx.TeamID, sbx.SandboxID, opts); err != nil {

packages/api/internal/orchestrator/evictor/evict.go

@@ -15,7 +15,7 @@ import (
 func TestEvictSandbox_ReasonByAction(t *testing.T) {
 	t.Parallel()
 
-	run := func(autoPause bool) sandbox.RemoveOpts {
+	run := func(autoPause, autoPauseFilesystemOnly bool) sandbox.RemoveOpts {
 		var got sandbox.RemoveOpts
 		called := false
 		e := &Evictor{
@@ -33,10 +33,11 @@ func TestEvictSandbox_ReasonByAction(t *testing.T) {
 		}
 
 		e.evictSandbox(context.Background(), sandbox.Sandbox{
-			SandboxID: "sbx",
-			TeamID:    uuid.New(),
-			AutoPause: autoPause,
-			EndTime:   time.Now(),
+			SandboxID:               "sbx",
+			TeamID:                  uuid.New(),
+			AutoPause:               autoPause,
+			AutoPauseFilesystemOnly: autoPauseFilesystemOnly,
+			EndTime:                 time.Now(),
 		})
 
 		require.True(t, called)
@@ -47,19 +48,48 @@ func TestEvictSandbox_ReasonByAction(t *testing.T) {
 	t.Run("kill carries timeout reason", func(t *testing.T) {
 		t.Parallel()
 
-		got := run(false)
+		got := run(false, false)
 
 		assert.Equal(t, sandbox.StateActionKill, got.Action)
 		assert.True(t, got.Eviction)
 		assert.Equal(t, sandbox.KillReasonTimeout, got.Reason)
 	})
 
+	t.Run("kill ignores the auto-pause snapshot kind", func(t *testing.T) {
+		t.Parallel()
+
+		// AutoPauseFilesystemOnly is meaningless without AutoPause; a kill must
+		// never carry it.
+		got := run(false, true)
+
+		assert.Equal(t, sandbox.StateActionKill, got.Action)
+		assert.False(t, got.FilesystemOnly)
+	})
+
 	t.Run("auto-pause carries no kill reason", func(t *testing.T) {
 		t.Parallel()
 
-		got := run(true)
+		got := run(true, false)
 
 		assert.Equal(t, sandbox.StateActionPause, got.Action)
 		assert.Empty(t, got.Reason)
 	})
+
+	t.Run("memory auto-pause is not filesystem-only", func(t *testing.T) {
+		t.Parallel()
+
+		got := run(true, false)
+
+		assert.Equal(t, sandbox.StateActionPause, got.Action)
+		assert.False(t, got.FilesystemOnly)
+	})
+
+	t.Run("filesystem-only auto-pause requests a filesystem-only snapshot", func(t *testing.T) {
+		t.Parallel()
+
+		got := run(true, true)
+
+		assert.Equal(t, sandbox.StateActionPause, got.Action)
+		assert.True(t, got.FilesystemOnly)
+	})
 }

packages/api/internal/orchestrator/evictor/evict_test.go

@@ -131,6 +131,7 @@ func (n *Node) GetSandboxes(ctx context.Context) ([]sandbox.Sandbox, error) {
 				n.ID,
 				n.ClusterID,
 				config.GetAutoPause(),
+				config.GetAutoPauseFilesystemOnly(),
 				autoResume,
 				config.EnvdAccessToken,     //nolint:protogetter // we need the nil check too
 				config.AllowInternetAccess, //nolint:protogetter // we need the nil check too

packages/api/internal/orchestrator/nodemanager/sandboxes.go

@@ -0,0 +1,79 @@
+package nodemanager
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/protobuf/types/known/emptypb"
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	"github.com/e2b-dev/infra/packages/api/internal/api"
+	"github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator"
+)
+
+// mockSandboxListClient implements orchestrator.SandboxServiceClient and returns
+// a canned List response, so GetSandboxes' proto->Sandbox reconstruction can be
+// tested without a live orchestrator.
+type mockSandboxListClient struct {
+	orchestrator.SandboxServiceClient
+
+	resp *orchestrator.SandboxListResponse
+}
+
+func (m *mockSandboxListClient) List(_ context.Context, _ *emptypb.Empty, _ ...grpc.CallOption) (*orchestrator.SandboxListResponse, error) {
+	return m.resp, nil
+}
+
+// TestGetSandboxes_RestoresAutoPauseFilesystemOnly verifies that the auto-pause
+// snapshot-kind policy round-trips through the orchestrator's SandboxConfig when
+// the API re-syncs its sandbox list (e.g. after a restart). The proto field
+// exists for exactly this path, so without it the policy would silently revert
+// to a memory auto-pause.
+func TestGetSandboxes_RestoresAutoPauseFilesystemOnly(t *testing.T) {
+	t.Parallel()
+
+	now := time.Now()
+	runningSandbox := func(id string, autoPauseFilesystemOnly bool) *orchestrator.RunningSandbox {
+		return &orchestrator.RunningSandbox{
+			StartTime: timestamppb.New(now),
+			EndTime:   timestamppb.New(now.Add(time.Hour)),
+			Config: &orchestrator.SandboxConfig{
+				SandboxId:               id,
+				TemplateId:              "tmpl",
+				BaseTemplateId:          "tmpl",
+				TeamId:                  uuid.NewString(),
+				BuildId:                 uuid.NewString(),
+				ExecutionId:             uuid.NewString(),
+				AutoPause:               true,
+				AutoPauseFilesystemOnly: autoPauseFilesystemOnly,
+			},
+		}
+	}
+
+	node := NewTestNode("test-node", api.NodeStatusReady, 0, 4)
+	node.SetSandboxClient(&mockSandboxListClient{
+		resp: &orchestrator.SandboxListResponse{
+			Sandboxes: []*orchestrator.RunningSandbox{
+				runningSandbox("fs-only", true),
+				runningSandbox("memory", false),
+			},
+		},
+	})
+
+	sandboxes, err := node.GetSandboxes(t.Context())
+	require.NoError(t, err)
+	require.Len(t, sandboxes, 2)
+
+	got := make(map[string]bool, len(sandboxes))
+	for _, sbx := range sandboxes {
+		got[sbx.SandboxID] = sbx.AutoPauseFilesystemOnly
+	}
+
+	assert.True(t, got["fs-only"], "filesystem-only auto-pause policy must survive an orchestrator re-sync")
+	assert.False(t, got["memory"], "memory auto-pause policy must survive an orchestrator re-sync")
+}

packages/api/internal/orchestrator/nodemanager/sandboxes_test.go

@@ -154,11 +154,12 @@ func buildUpsertSnapshotParams(sbx sandbox.Sandbox, node *nodemanager.Node, file
 		AllowInternetAccess: sbx.AllowInternetAccess,
 		AutoPause:           sbx.AutoPause,
 		Config: &types.PausedSandboxConfig{
-			Version:        types.PausedSandboxConfigVersion,
-			Network:        sbx.Network,
-			AutoResume:     sbx.AutoResume,
-			VolumeMounts:   sbx.VolumeMounts,
-			FilesystemOnly: filesystemOnly,
+			Version:                 types.PausedSandboxConfigVersion,
+			Network:                 sbx.Network,
+			AutoResume:              sbx.AutoResume,
+			VolumeMounts:            sbx.VolumeMounts,
+			FilesystemOnly:          filesystemOnly,
+			AutoPauseFilesystemOnly: sbx.AutoPauseFilesystemOnly,
 		},
 		OriginNodeID: node.ID,
 		Status:       types.BuildStatusSnapshotting,