e2b-dev · levb · May 5, 2026 · Feb 27, 2026 · Feb 27, 2026 · Mar 1, 2026
@@ -1,6 +1,24 @@
 name: "Build Sandbox Template"
 description: "Builds the Firecracker sandbox template."
 
+inputs:
+  compress_enabled:
+    description: "Enable compression (true/false)"
+    required: false
+    default: "false"
+  compress_type:
+    description: "Compression type (zstd, lz4)"
+    required: false
+    default: ""
+  compress_level:
+    description: "Compression level (zstd: 1=fastest, 2=default; lz4: 0)"
+    required: false
+    default: ""
+  compress_workers:
+    description: "Number of frame encode workers"
+    required: false
+    default: ""
+
 runs:
   using: "composite"
   steps:
@@ -9,6 +27,10 @@ runs:
         TEMPLATE_ID: "2j6ly824owf4awgai1xo"
         KERNEL_VERSION: "vmlinux-6.1.158"
         FIRECRACKER_VERSION: "v1.14.1_458ca91"
+        COMPRESS_ENABLED: ${{ inputs.compress_enabled }}
+        COMPRESS_TYPE: ${{ inputs.compress_type }}
+        COMPRESS_LEVEL: ${{ inputs.compress_level }}
+        COMPRESS_FRAME_ENCODE_WORKERS: ${{ inputs.compress_workers }}
       run: |
         # Generate an unique build ID for the template for this run
         export BUILD_ID=$(uuidgen)
@@ -17,6 +39,10 @@ runs:
 
         echo "TESTS_SANDBOX_TEMPLATE_ID=${TEMPLATE_ID}" >> .env.test
         echo "TESTS_SANDBOX_BUILD_ID=${BUILD_ID}" >> .env.test
+        echo "COMPRESS_ENABLED=${COMPRESS_ENABLED}" >> .env.test
+        echo "COMPRESS_TYPE=${COMPRESS_TYPE}" >> .env.test
+        echo "COMPRESS_LEVEL=${COMPRESS_LEVEL}" >> .env.test
+        echo "COMPRESS_FRAME_ENCODE_WORKERS=${COMPRESS_FRAME_ENCODE_WORKERS}" >> .env.test
 
         sudo -E make -C packages/orchestrator build-template \
           ARTIFACTS_REGISTRY_PROVIDER=Local \

@@ -1,6 +1,24 @@
 name: "Start Services"
 description: "Sets up and starts the required services, including PostgreSQL."
 
+inputs:
+  compress_enabled:
+    description: "Enable compression (true/false)"
+    required: false
+    default: "false"
+  compress_type:
+    description: "Compression type (zstd, lz4)"
+    required: false
+    default: ""
+  compress_level:
+    description: "Compression level (zstd: 1=fastest, 2=default; lz4: 0)"
+    required: false
+    default: ""
+  compress_workers:
+    description: "Number of frame encode workers"
+    required: false
+    default: ""
+
 runs:
   using: "composite"
   steps:
@@ -107,6 +125,11 @@ runs:
         API_INTERNAL_GRPC_ADDRESS: "localhost:5009"
         DEFAULT_PERSISTENT_VOLUME_TYPE: "test-volume-type"
         SANDBOX_STORAGE_BACKEND: "redis"
+        COMPRESS_ENABLED: ${{ inputs.compress_enabled }}
+        COMPRESS_TYPE: ${{ inputs.compress_type }}
+        COMPRESS_LEVEL: ${{ inputs.compress_level }}
+        COMPRESS_FRAME_ENCODE_WORKERS: ${{ inputs.compress_workers }}
+        E2B_DEBUG: "true"
       run: |
         mkdir -p $SHARED_CHUNK_CACHE_PATH
         mkdir -p ~/logs

@@ -10,12 +10,31 @@ on:
     secrets:
       CODECOV_TOKEN: { required: false }
 jobs:
-  integration_tests:
+  run:
     runs-on: infra-tests
     timeout-minutes: 30
     permissions:
       contents: read
       id-token: write
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: uncompressed
+            compress_enabled: "false"
+            compress_type: ""
+            compress_level: ""
+            compress_workers: ""
+          - name: zstd1
+            compress_enabled: "true"
+            compress_type: "zstd"
+            compress_level: "1"
+            compress_workers: "8"
+          - name: lz4
+            compress_enabled: "true"
+            compress_type: "lz4"
+            compress_level: "0"
+            compress_workers: "8"
     env:
       # Surfaced as env so upload steps can gate on presence (skipped on fork PRs).
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
@@ -32,9 +51,19 @@ jobs:
 
       - name: Build Template
         uses: ./.github/actions/build-sandbox-template
+        with:
+          compress_enabled: ${{ matrix.compress_enabled }}
+          compress_type: ${{ matrix.compress_type }}
+          compress_level: ${{ matrix.compress_level }}
+          compress_workers: ${{ matrix.compress_workers }}
 
       - name: Start Services
         uses: ./.github/actions/start-services
+        with:
+          compress_enabled: ${{ matrix.compress_enabled }}
+          compress_type: ${{ matrix.compress_type }}
+          compress_level: ${{ matrix.compress_level }}
+          compress_workers: ${{ matrix.compress_workers }}
 
       - name: Run Integration Tests
         env:
@@ -80,7 +109,7 @@ jobs:
         if: ${{ always() && inputs.publish == true }}
         uses: actions/upload-artifact@v6
         with:
-          name: Integration Tests Results
+          name: Integration Tests Results (${{ matrix.name }})
           path: ./tests/integration/test-results.xml
 
       - name: Upload test results to Codecov
@@ -96,5 +125,18 @@ jobs:
         if: ${{ always() && inputs.publish == true }}
         uses: actions/upload-artifact@v6
         with:
-          name: Service Logs
+          name: Service Logs (${{ matrix.name }})
           path: ~/logs/*.log
+
+  integration_tests:
+    needs: run
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Aggregate matrix result
+        run: |
+          if [[ "${{ needs.run.result }}" != "success" ]]; then
+            echo "matrix result: ${{ needs.run.result }}"
+            exit 1
+          fi
+          echo "all matrix shards succeeded"
@@ -39,34 +39,38 @@ packages:
     interfaces:
       featureFlagsClient:
         config:
-          dir: packages/shared/pkg/storage/mocks
-          filename: mockfeatureflagsclient.go
-          pkgname: storagemocks
+          dir: packages/shared/pkg/storage
+          filename: mock_featureflagsclient.go
+          pkgname: storage
+          inpackage: true
           structname: MockFeatureFlagsClient
       Blob:
         config:
-          dir: packages/shared/pkg/storage/mocks
-          filename: mockobjectprovider.go
-          pkgname: storagemocks
+          dir: packages/shared/pkg/storage
+          filename: mock_blob.go
+          pkgname: storage
+          inpackage: true
       Seekable:
         config:
-          dir: packages/shared/pkg/storage/mocks
-          filename: mockseekableobjectprovider.go
-          pkgname: storagemocks
+          dir: packages/shared/pkg/storage
+          filename: mock_seekable.go
+          pkgname: storage
+          inpackage: true
       StorageProvider:
         config:
-          dir: packages/shared/pkg/storage/mocks/provider
-          filename: mockstorageprovider.go
-          pkgname: providermocks
-
+          dir: packages/shared/pkg/storage
+          filename: mock_storageprovider.go
+          pkgname: storage
+          inpackage: true
 
   io:
     interfaces:
       Reader:
         config:
-          dir: packages/shared/pkg/storage/mocks
-          filename: mockioreader.go
-          pkgname: storagemocks
+          dir: packages/shared/pkg/storage
+          filename: mock_ioreader.go
+          pkgname: storage
+          inpackage: true
 
   github.com/e2b-dev/infra/packages/shared/pkg/utils:
     interfaces:
@@ -76,6 +80,14 @@ packages:
           filename: mocks_test.go
           pkgname: utils
 
+  github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/build:
+    interfaces:
+      Diff:
+        config:
+          dir: packages/orchestrator/pkg/sandbox/build/mocks
+          filename: mockdiff.go
+          pkgname: buildmocks
+
   github.com/e2b-dev/infra/packages/api/internal/handlers:
     interfaces:
       featureFlagsClient:

@@ -135,6 +135,10 @@ build-template: fetch-busybox
 	GCP_PROJECT_ID=$(GCP_PROJECT_ID) \
 	GCP_DOCKER_REPOSITORY_NAME=$(GCP_DOCKER_REPOSITORY_NAME) \
 	GCP_REGION=$(GCP_REGION) \
+	COMPRESS_ENABLED=$(COMPRESS_ENABLED) \
+	COMPRESS_TYPE=$(COMPRESS_TYPE) \
+	COMPRESS_LEVEL=$(COMPRESS_LEVEL) \
+	COMPRESS_FRAME_ENCODE_WORKERS=$(COMPRESS_FRAME_ENCODE_WORKERS) \
 	ENVIRONMENT=local \
 	go run cmd/create-build/main.go \
 	-template $(TEMPLATE_ID) \

@@ -275,6 +275,7 @@ func BenchmarkBaseImageLaunch(b *testing.B) {
 		sandboxes,
 		templateCache,
 		buildMetrics,
+		nil,
 	)
 
 	buildPath := filepath.Join(os.Getenv("LOCAL_TEMPLATE_STORAGE_BASE_PATH"), buildID, "rootfs.ext4")

@@ -324,6 +324,7 @@ func BenchmarkConcurrentResume(b *testing.B) {
 		config.BuilderConfig, l, featureFlags, sandboxFactory,
 		persistenceTemplate, persistenceBuild, artifactRegistry,
 		dockerhubRepository, sandboxProxy, sandboxes, templateCache, buildMetrics,
+		nil,
 	)
 
 	// build template if not cached

@@ -2,17 +2,19 @@ syntax = "proto3";
 
 option go_package = "https://github.com/e2b-dev/infra/orchestrator";
 
-// ChunkService allows orchestrators to serve snapshot files directly from
-// their local cache to peer orchestrators, bypassing GCS during hot resumes.
+// ChunkService allows orchestrators to serve snapshot files directly from their
+// local cache to peer orchestrators, bypassing remote storage during hot
+// resumes.
 
 // PeerAvailability carries the routing decision included in every response.
 // When neither flag is set, the file is available in the peer's local cache.
 message PeerAvailability {
-  // not_available is true when the file is not in the local cache.
-  // The caller should fall back to GCS.
+  // not_available is true when the file is not in the local cache. The caller
+  // should fall back to remote storage.
   bool not_available = 1;
-  // use_storage is true when the GCS upload has completed and the caller
-  // should switch to reading from GCS/NFS directly instead of this peer.
+  // use_storage is true when the remote storage upload has completed and the
+  // caller should switch to reading from remote storage directly instead of
+  // this peer.
   bool use_storage = 2;
 }
 

@@ -320,10 +320,18 @@ func doBuild(
 	buildMetrics, _ := metrics.NewBuildMetrics(noop.MeterProvider{})
 	sandboxFactory := sandbox.NewFactory(c.BuilderConfig, networkPool, devicePool, featureFlags, hoststats.NewNoopDelivery(), cgroup.NewNoopManager(), network.NewNoopEgressProxy(), sandboxes)
 
+	// Layered V4 builds need the upload coordinator so child layers wait on
+	// their parents' header finalization. Redis is nil (CLI is single-host —
+	// no cross-orch signaling needed); local same-orch coordination via
+	// futures is what matters here.
+	uploads := sandbox.NewUploads(templateCache, persistenceTemplate, nil)
+	defer uploads.Stop()
+
 	builder := build.NewBuilder(
 		builderConfig, l, featureFlags, sandboxFactory,
 		persistenceTemplate, persistenceBuild, artifactRegistry,
 		dockerhubRepo, sandboxProxy, sandboxes, templateCache, buildMetrics,
+		uploads,
 	)
 
 	l = l.With(zap.String("envID", templateID)).With(zap.String("buildID", buildID))

@@ -656,21 +656,23 @@ func (r *runner) pauseOnce(ctx context.Context, opts pauseOptions, verbose bool)
 
 	// Only upload when not in benchmark mode (verbose = true means single run)
 	if verbose {
-		paths := storage.Paths{BuildID: opts.newBuildID}
 		if opts.isRemoteStorage {
 			fmt.Println("📤 Uploading snapshot...")
-			if err := snapshot.Upload(ctx, r.storage, paths, nil); err != nil {
-				return timings, fmt.Errorf("failed to upload snapshot: %w", err)
-			}
-			fmt.Println("✅ Snapshot uploaded successfully")
 		} else {
 			fmt.Println("💾 Saving snapshot to local storage...")
-			if err := snapshot.Upload(ctx, r.storage, paths, nil); err != nil {
-				return timings, fmt.Errorf("failed to save snapshot: %w", err)
-			}
-			fmt.Println("✅ Snapshot saved successfully")
 		}
 
+		upload, err := sandbox.NewUpload(ctx, nil, snapshot, r.storage, storage.CompressConfig{}, nil, "", nil)
+		if err != nil {
+			return timings, fmt.Errorf("failed to prepare upload: %w", err)
+		}
+
+		if err := upload.Run(ctx); err != nil {
+			return timings, fmt.Errorf("failed to upload snapshot: %w", err)
+		}
+
+		fmt.Println("✅ Snapshot uploaded successfully")
+
 		fmt.Printf("\n✅ Build finished: %s\n", opts.newBuildID)
 		printArtifactSizes(opts.storagePath, opts.newBuildID)
 

@@ -237,6 +237,7 @@ func newTestInfra(t *testing.T, ctx context.Context) *testInfra {
 		builderConfig, l, flags, factory,
 		persistenceTemplate, persistenceBuild, artifactRegistry,
 		dockerhubRepo, sandboxProxy, sandboxes, templateCache, buildMetrics,
+		nil,
 	)
 
 	return ti

@@ -548,6 +548,13 @@ func run(config cfg.Config, opts Options) (success bool) {
 	builder := chrooted.NewBuilder(config)
 	volumeService := volumes.New(config, builder)
 
+	uploads := sandbox.NewUploads(templateCache, persistence, redisClient)
+	closers = append(closers, closer{"pending uploads", func(context.Context) error {
+		uploads.Stop()
+
+		return nil
+	}})
+
 	orchestratorService, err := server.New(server.ServiceConfig{
 		Config:           config,
 		SandboxFactory:   sandboxFactory,
@@ -561,10 +568,14 @@ func run(config cfg.Config, opts Options) (success bool) {
 		FeatureFlags:     featureFlags,
 		SbxEventsService: events.NewEventsService(sbxEventsDeliveryTargets),
 		PeerRegistry:     peerRegistry,
+		Uploads:          uploads,
 	})
 	if err != nil {
 		logger.L().Fatal(ctx, "failed to create orchestrator server", zap.Error(err))
 	}
+	closers = append(closers, closer{"orchestrator server", func(context.Context) error {
+		return orchestratorService.Close()
+	}})
 
 	// template manager sandbox logger
 	tmplSbxLoggerExternal := sbxlogger.NewLogger(
@@ -639,6 +650,7 @@ func run(config cfg.Config, opts Options) (success bool) {
 			templateCache,
 			persistence,
 			buildPersistence,
+			uploads,
 		)
 		if err != nil {
 			logger.L().Fatal(ctx, "failed to create template manager", zap.Error(err))