Skip to content

Commit 08810db

Browse files
levbclaudegithub-actions[bot]dobrac
authored
feature: Compression for memfile/rootfs assets (#2034)
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Co-authored-by: Jakub Dobry <jakub.dobry8@gmail.com>
1 parent 758e726 commit 08810db

103 files changed

Lines changed: 7477 additions & 3189 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/actions/build-sandbox-template/action.yml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,24 @@
11
name: "Build Sandbox Template"
22
description: "Builds the Firecracker sandbox template."
33

4+
inputs:
5+
compress_enabled:
6+
description: "Enable compression (true/false)"
7+
required: false
8+
default: "false"
9+
compress_type:
10+
description: "Compression type (zstd, lz4)"
11+
required: false
12+
default: ""
13+
compress_level:
14+
description: "Compression level (zstd: 1=fastest, 2=default; lz4: 0)"
15+
required: false
16+
default: ""
17+
compress_workers:
18+
description: "Number of frame encode workers"
19+
required: false
20+
default: ""
21+
422
runs:
523
using: "composite"
624
steps:
@@ -9,6 +27,10 @@ runs:
927
TEMPLATE_ID: "2j6ly824owf4awgai1xo"
1028
KERNEL_VERSION: "vmlinux-6.1.158"
1129
FIRECRACKER_VERSION: "v1.14.1_458ca91"
30+
COMPRESS_ENABLED: ${{ inputs.compress_enabled }}
31+
COMPRESS_TYPE: ${{ inputs.compress_type }}
32+
COMPRESS_LEVEL: ${{ inputs.compress_level }}
33+
COMPRESS_FRAME_ENCODE_WORKERS: ${{ inputs.compress_workers }}
1234
run: |
1335
# Generate an unique build ID for the template for this run
1436
export BUILD_ID=$(uuidgen)
@@ -17,6 +39,10 @@ runs:
1739
1840
echo "TESTS_SANDBOX_TEMPLATE_ID=${TEMPLATE_ID}" >> .env.test
1941
echo "TESTS_SANDBOX_BUILD_ID=${BUILD_ID}" >> .env.test
42+
echo "COMPRESS_ENABLED=${COMPRESS_ENABLED}" >> .env.test
43+
echo "COMPRESS_TYPE=${COMPRESS_TYPE}" >> .env.test
44+
echo "COMPRESS_LEVEL=${COMPRESS_LEVEL}" >> .env.test
45+
echo "COMPRESS_FRAME_ENCODE_WORKERS=${COMPRESS_FRAME_ENCODE_WORKERS}" >> .env.test
2046
2147
sudo -E make -C packages/orchestrator build-template \
2248
ARTIFACTS_REGISTRY_PROVIDER=Local \

‎.github/actions/start-services/action.yml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,24 @@
11
name: "Start Services"
22
description: "Sets up and starts the required services, including PostgreSQL."
33

4+
inputs:
5+
compress_enabled:
6+
description: "Enable compression (true/false)"
7+
required: false
8+
default: "false"
9+
compress_type:
10+
description: "Compression type (zstd, lz4)"
11+
required: false
12+
default: ""
13+
compress_level:
14+
description: "Compression level (zstd: 1=fastest, 2=default; lz4: 0)"
15+
required: false
16+
default: ""
17+
compress_workers:
18+
description: "Number of frame encode workers"
19+
required: false
20+
default: ""
21+
422
runs:
523
using: "composite"
624
steps:
@@ -107,6 +125,11 @@ runs:
107125
API_INTERNAL_GRPC_ADDRESS: "localhost:5009"
108126
DEFAULT_PERSISTENT_VOLUME_TYPE: "test-volume-type"
109127
SANDBOX_STORAGE_BACKEND: "redis"
128+
COMPRESS_ENABLED: ${{ inputs.compress_enabled }}
129+
COMPRESS_TYPE: ${{ inputs.compress_type }}
130+
COMPRESS_LEVEL: ${{ inputs.compress_level }}
131+
COMPRESS_FRAME_ENCODE_WORKERS: ${{ inputs.compress_workers }}
132+
E2B_DEBUG: "true"
110133
run: |
111134
mkdir -p $SHARED_CHUNK_CACHE_PATH
112135
mkdir -p ~/logs

‎.github/workflows/integration_tests.yml‎

Lines changed: 45 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,31 @@ on:
1010
secrets:
1111
CODECOV_TOKEN: { required: false }
1212
jobs:
13-
integration_tests:
13+
run:
1414
runs-on: infra-tests
1515
timeout-minutes: 30
1616
permissions:
1717
contents: read
1818
id-token: write
19+
strategy:
20+
fail-fast: false
21+
matrix:
22+
include:
23+
- name: uncompressed
24+
compress_enabled: "false"
25+
compress_type: ""
26+
compress_level: ""
27+
compress_workers: ""
28+
- name: zstd1
29+
compress_enabled: "true"
30+
compress_type: "zstd"
31+
compress_level: "1"
32+
compress_workers: "8"
33+
- name: lz4
34+
compress_enabled: "true"
35+
compress_type: "lz4"
36+
compress_level: "0"
37+
compress_workers: "8"
1938
env:
2039
# Surfaced as env so upload steps can gate on presence (skipped on fork PRs).
2140
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
@@ -32,9 +51,19 @@ jobs:
3251

3352
- name: Build Template
3453
uses: ./.github/actions/build-sandbox-template
54+
with:
55+
compress_enabled: ${{ matrix.compress_enabled }}
56+
compress_type: ${{ matrix.compress_type }}
57+
compress_level: ${{ matrix.compress_level }}
58+
compress_workers: ${{ matrix.compress_workers }}
3559

3660
- name: Start Services
3761
uses: ./.github/actions/start-services
62+
with:
63+
compress_enabled: ${{ matrix.compress_enabled }}
64+
compress_type: ${{ matrix.compress_type }}
65+
compress_level: ${{ matrix.compress_level }}
66+
compress_workers: ${{ matrix.compress_workers }}
3867

3968
- name: Run Integration Tests
4069
env:
@@ -80,7 +109,7 @@ jobs:
80109
if: ${{ always() && inputs.publish == true }}
81110
uses: actions/upload-artifact@v6
82111
with:
83-
name: Integration Tests Results
112+
name: Integration Tests Results (${{ matrix.name }})
84113
path: ./tests/integration/test-results.xml
85114

86115
- name: Upload test results to Codecov
@@ -96,5 +125,18 @@ jobs:
96125
if: ${{ always() && inputs.publish == true }}
97126
uses: actions/upload-artifact@v6
98127
with:
99-
name: Service Logs
128+
name: Service Logs (${{ matrix.name }})
100129
path: ~/logs/*.log
130+
131+
integration_tests:
132+
needs: run
133+
if: always()
134+
runs-on: ubuntu-latest
135+
steps:
136+
- name: Aggregate matrix result
137+
run: |
138+
if [[ "${{ needs.run.result }}" != "success" ]]; then
139+
echo "matrix result: ${{ needs.run.result }}"
140+
exit 1
141+
fi
142+
echo "all matrix shards succeeded"

‎.mockery.yaml‎

Lines changed: 28 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -39,34 +39,38 @@ packages:
3939
interfaces:
4040
featureFlagsClient:
4141
config:
42-
dir: packages/shared/pkg/storage/mocks
43-
filename: mockfeatureflagsclient.go
44-
pkgname: storagemocks
42+
dir: packages/shared/pkg/storage
43+
filename: mock_featureflagsclient.go
44+
pkgname: storage
45+
inpackage: true
4546
structname: MockFeatureFlagsClient
4647
Blob:
4748
config:
48-
dir: packages/shared/pkg/storage/mocks
49-
filename: mockobjectprovider.go
50-
pkgname: storagemocks
49+
dir: packages/shared/pkg/storage
50+
filename: mock_blob.go
51+
pkgname: storage
52+
inpackage: true
5153
Seekable:
5254
config:
53-
dir: packages/shared/pkg/storage/mocks
54-
filename: mockseekableobjectprovider.go
55-
pkgname: storagemocks
55+
dir: packages/shared/pkg/storage
56+
filename: mock_seekable.go
57+
pkgname: storage
58+
inpackage: true
5659
StorageProvider:
5760
config:
58-
dir: packages/shared/pkg/storage/mocks/provider
59-
filename: mockstorageprovider.go
60-
pkgname: providermocks
61-
61+
dir: packages/shared/pkg/storage
62+
filename: mock_storageprovider.go
63+
pkgname: storage
64+
inpackage: true
6265

6366
io:
6467
interfaces:
6568
Reader:
6669
config:
67-
dir: packages/shared/pkg/storage/mocks
68-
filename: mockioreader.go
69-
pkgname: storagemocks
70+
dir: packages/shared/pkg/storage
71+
filename: mock_ioreader.go
72+
pkgname: storage
73+
inpackage: true
7074

7175
github.com/e2b-dev/infra/packages/shared/pkg/utils:
7276
interfaces:
@@ -76,6 +80,14 @@ packages:
7680
filename: mocks_test.go
7781
pkgname: utils
7882

83+
github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/build:
84+
interfaces:
85+
Diff:
86+
config:
87+
dir: packages/orchestrator/pkg/sandbox/build/mocks
88+
filename: mockdiff.go
89+
pkgname: buildmocks
90+
7991
github.com/e2b-dev/infra/packages/api/internal/handlers:
8092
interfaces:
8193
featureFlagsClient:

‎packages/orchestrator/Makefile‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,10 @@ build-template: fetch-busybox
135135
GCP_PROJECT_ID=$(GCP_PROJECT_ID) \
136136
GCP_DOCKER_REPOSITORY_NAME=$(GCP_DOCKER_REPOSITORY_NAME) \
137137
GCP_REGION=$(GCP_REGION) \
138+
COMPRESS_ENABLED=$(COMPRESS_ENABLED) \
139+
COMPRESS_TYPE=$(COMPRESS_TYPE) \
140+
COMPRESS_LEVEL=$(COMPRESS_LEVEL) \
141+
COMPRESS_FRAME_ENCODE_WORKERS=$(COMPRESS_FRAME_ENCODE_WORKERS) \
138142
ENVIRONMENT=local \
139143
go run cmd/create-build/main.go \
140144
-template $(TEMPLATE_ID) \

‎packages/orchestrator/benchmarks/benchmark_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -275,6 +275,7 @@ func BenchmarkBaseImageLaunch(b *testing.B) {
275275
sandboxes,
276276
templateCache,
277277
buildMetrics,
278+
nil,
278279
)
279280

280281
buildPath := filepath.Join(os.Getenv("LOCAL_TEMPLATE_STORAGE_BASE_PATH"), buildID, "rootfs.ext4")

‎packages/orchestrator/benchmarks/concurrent_benchmark_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -324,6 +324,7 @@ func BenchmarkConcurrentResume(b *testing.B) {
324324
config.BuilderConfig, l, featureFlags, sandboxFactory,
325325
persistenceTemplate, persistenceBuild, artifactRegistry,
326326
dockerhubRepository, sandboxProxy, sandboxes, templateCache, buildMetrics,
327+
nil,
327328
)
328329

329330
// build template if not cached

‎packages/orchestrator/chunks.proto‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,19 @@ syntax = "proto3";
22

33
option go_package = "https://github.com/e2b-dev/infra/orchestrator";
44

5-
// ChunkService allows orchestrators to serve snapshot files directly from
6-
// their local cache to peer orchestrators, bypassing GCS during hot resumes.
5+
// ChunkService allows orchestrators to serve snapshot files directly from their
6+
// local cache to peer orchestrators, bypassing remote storage during hot
7+
// resumes.
78

89
// PeerAvailability carries the routing decision included in every response.
910
// When neither flag is set, the file is available in the peer's local cache.
1011
message PeerAvailability {
11-
// not_available is true when the file is not in the local cache.
12-
// The caller should fall back to GCS.
12+
// not_available is true when the file is not in the local cache. The caller
13+
// should fall back to remote storage.
1314
bool not_available = 1;
14-
// use_storage is true when the GCS upload has completed and the caller
15-
// should switch to reading from GCS/NFS directly instead of this peer.
15+
// use_storage is true when the remote storage upload has completed and the
16+
// caller should switch to reading from remote storage directly instead of
17+
// this peer.
1618
bool use_storage = 2;
1719
}
1820

‎packages/orchestrator/cmd/create-build/main.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -320,10 +320,18 @@ func doBuild(
320320
buildMetrics, _ := metrics.NewBuildMetrics(noop.MeterProvider{})
321321
sandboxFactory := sandbox.NewFactory(c.BuilderConfig, networkPool, devicePool, featureFlags, hoststats.NewNoopDelivery(), cgroup.NewNoopManager(), network.NewNoopEgressProxy(), sandboxes)
322322

323+
// Layered V4 builds need the upload coordinator so child layers wait on
324+
// their parents' header finalization. Redis is nil (CLI is single-host —
325+
// no cross-orch signaling needed); local same-orch coordination via
326+
// futures is what matters here.
327+
uploads := sandbox.NewUploads(templateCache, persistenceTemplate, nil)
328+
defer uploads.Stop()
329+
323330
builder := build.NewBuilder(
324331
builderConfig, l, featureFlags, sandboxFactory,
325332
persistenceTemplate, persistenceBuild, artifactRegistry,
326333
dockerhubRepo, sandboxProxy, sandboxes, templateCache, buildMetrics,
334+
uploads,
327335
)
328336

329337
l = l.With(zap.String("envID", templateID)).With(zap.String("buildID", buildID))

‎packages/orchestrator/cmd/resume-build/main.go‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -656,21 +656,23 @@ func (r *runner) pauseOnce(ctx context.Context, opts pauseOptions, verbose bool)
656656

657657
// Only upload when not in benchmark mode (verbose = true means single run)
658658
if verbose {
659-
paths := storage.Paths{BuildID: opts.newBuildID}
660659
if opts.isRemoteStorage {
661660
fmt.Println("📤 Uploading snapshot...")
662-
if err := snapshot.Upload(ctx, r.storage, paths, nil); err != nil {
663-
return timings, fmt.Errorf("failed to upload snapshot: %w", err)
664-
}
665-
fmt.Println("✅ Snapshot uploaded successfully")
666661
} else {
667662
fmt.Println("💾 Saving snapshot to local storage...")
668-
if err := snapshot.Upload(ctx, r.storage, paths, nil); err != nil {
669-
return timings, fmt.Errorf("failed to save snapshot: %w", err)
670-
}
671-
fmt.Println("✅ Snapshot saved successfully")
672663
}
673664

665+
upload, err := sandbox.NewUpload(ctx, nil, snapshot, r.storage, storage.CompressConfig{}, nil, "", nil)
666+
if err != nil {
667+
return timings, fmt.Errorf("failed to prepare upload: %w", err)
668+
}
669+
670+
if err := upload.Run(ctx); err != nil {
671+
return timings, fmt.Errorf("failed to upload snapshot: %w", err)
672+
}
673+
674+
fmt.Println("✅ Snapshot uploaded successfully")
675+
674676
fmt.Printf("\n✅ Build finished: %s\n", opts.newBuildID)
675677
printArtifactSizes(opts.storagePath, opts.newBuildID)
676678

0 commit comments

Comments
 (0)