Skip to content

Commit 9e372a6

Browse files
committed
Add OCI image unpack and run
Turn a pulled image into something `elfuse` can actually execute: - vendor decode-only zstd v1.5.6 (compression / dictBuilder / legacy paths excluded; only oci/decompress.c includes the header) - tar reader: ustar + GNU long-name records, used by layer apply - decompression dispatch: gzip via system zlib, zstd via vendored decode-only build, dispatched by layer media type - layer applier with whiteout-aware merge: typeflag '1' (hardlink), '2' (symlink), '5' (directory), `.wh.*` markers, symlink-escape containment - per-image sysroot on a case-sensitive APFS sparsebundle (hdiutil-provisioned, image_layout v1) - per-run rootfs via clonefile(2) on top of the sysroot - `oci unpack` and `oci clone` subcommands that exercise the above - `oci inspect` extended with the image-config runtime block (Entrypoint / Cmd / Env / WorkingDir / User) - runspec resolver merging image-config defaults with CLI Entrypoint / Cmd / Env / WorkingDir / User overrides - PATH resolver that walks the guest /usr/local/sbin..:/sbin chain inside the sysroot (no host PATH leakage) - `elfuse_launch` extraction from main.c so the elfuse runtime can be reused by both legacy ./binary mode and the new `oci run` - `oci run` subcommand that ties pull -> unpack -> clone -> launch - `oci-layout` 1.0.0 marker at the store root - migrate store pins from `refs/<name>` flat files to a single `index.json` (OCI image-layout 1.0); auto-migrate on store open
1 parent 824ca02 commit 9e372a6

87 files changed

Lines changed: 36357 additions & 296 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.gitignore

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,11 @@ build/
22
archive/
33
# externals/ holds downloaded fixtures (kernel, rootfs, packages) that are
44
# fetched on demand; tracking them in git would balloon the repo. The
5-
# vendored cJSON tree is an exception: it ships with the source so the
6-
# OCI parser builds out of the box.
5+
# vendored cJSON and zstd trees are exceptions: they ship with the source
6+
# so the OCI parser and layer unpacker build out of the box.
77
externals/*
88
!externals/cjson/
9+
!externals/zstd/
910
lib/modules/
1011
*.o
1112
*.bin

Makefile

Lines changed: 132 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ SRCS := \
2424
core/stack.c \
2525
core/vdso.c \
2626
core/bootstrap.c \
27+
core/launch.c \
2728
core/sysroot.c \
2829
runtime/thread.c \
2930
runtime/futex.c \
@@ -74,7 +75,17 @@ SRCS := \
7475
oci/fetch.c \
7576
oci/store.c \
7677
oci/pull.c \
77-
oci/inspect.c
78+
oci/inspect.c \
79+
oci/tar.c \
80+
oci/decompress.c \
81+
oci/layer-meta.c \
82+
oci/layer-apply.c \
83+
oci/volume.c \
84+
oci/clone-rootfs.c \
85+
oci/unpack.c \
86+
oci/runspec.c \
87+
oci/path-resolve.c \
88+
oci/run.c
7889

7990
SRCS := $(addprefix src/,$(SRCS))
8091
OBJS := $(patsubst src/%.c,$(BUILD_DIR)/%.o,$(SRCS))
@@ -91,10 +102,34 @@ $(CJSON_OBJ): $(CJSON_DIR)/cJSON.c $(CJSON_DIR)/cJSON.h | $(BUILD_DIR)
91102
@echo " CC $<"
92103
$(Q)$(CC) $(CFLAGS) -c -o $@ $<
93104

105+
# Vendored zstd v1.5.6 (decode-only). Phase 2 OCI layer unpack consumes
106+
# zstd-compressed layer media types. Compression, dictBuilder, deprecated,
107+
# and legacy v01-v06 paths are NOT vendored; do not call ZSTD_compress*.
108+
# Only src/oci/decompress.c includes externals/zstd/lib/zstd.h.
109+
ZSTD_DIR := externals/zstd
110+
ZSTD_SRCS := $(wildcard $(ZSTD_DIR)/lib/common/*.c) \
111+
$(wildcard $(ZSTD_DIR)/lib/decompress/*.c)
112+
ZSTD_OBJS := $(patsubst $(ZSTD_DIR)/%.c,$(BUILD_DIR)/externals/zstd/%.o,$(ZSTD_SRCS))
113+
OBJS += $(ZSTD_OBJS)
114+
115+
ZSTD_CFLAGS := -DZSTD_DISABLE_ASM=1 -DZSTD_LEGACY_SUPPORT=0 \
116+
-DZSTD_MULTITHREAD=0 -DZSTDLIB_VISIBILITY= \
117+
-Wno-pedantic -Wno-shadow -Wno-strict-prototypes \
118+
-Wno-missing-prototypes -Wno-unused-parameter \
119+
-Wno-cast-align -Wno-implicit-fallthrough \
120+
-I$(ZSTD_DIR)/lib -I$(ZSTD_DIR)/lib/common
121+
122+
$(BUILD_DIR)/externals/zstd/%.o: $(ZSTD_DIR)/%.c | $(BUILD_DIR)
123+
@mkdir -p $(dir $@)
124+
@echo " CC $<"
125+
$(Q)$(CC) $(CFLAGS) $(ZSTD_CFLAGS) -c -o $@ $<
126+
94127
DISPATCH_MANIFEST := src/syscall/dispatch.tbl
95128
DISPATCH_GENERATOR := scripts/gen-syscall-dispatch.py
96129
DISPATCH_HEADER := $(BUILD_DIR)/dispatch.h
97-
HVF_LDFLAGS := -framework Hypervisor -arch arm64 -lcurl
130+
# -lz: gzip-compressed OCI layers route through zlib (system library).
131+
# -lcurl: HTTPS fetch for the Phase 1 oci pull path.
132+
HVF_LDFLAGS := -framework Hypervisor -arch arm64 -lcurl -lz
98133

99134
# Generated headers under build/ that must exist before compiling sources that
100135
# include them.
@@ -198,8 +233,9 @@ $(BUILD_DIR)/test-oci-fetch: $(BUILD_DIR)/test-oci-fetch.o $(BUILD_DIR)/lib/oci-
198233
$(Q)$(CC) $(CFLAGS) -o $@ $^ -lcurl -lpthread $(OPENSSL_LDFLAGS)
199234

200235
## Build the OCI local store unit test (native macOS, no HVF). Pure C; links
201-
## against the store wrapper plus its blob-store and digest dependencies.
202-
$(BUILD_DIR)/test-oci-store: $(BUILD_DIR)/test-oci-store.o $(BUILD_DIR)/oci/store.o $(BUILD_DIR)/oci/blob-store.o $(BUILD_DIR)/oci/digest.o $(BUILD_DIR)/oci/ref.o | $(BUILD_DIR)
236+
## against the store wrapper plus its blob-store, digest, and cJSON deps.
237+
## cJSON is required because store.c now reads / writes index.json.
238+
$(BUILD_DIR)/test-oci-store: $(BUILD_DIR)/test-oci-store.o $(BUILD_DIR)/oci/store.o $(BUILD_DIR)/oci/blob-store.o $(BUILD_DIR)/oci/digest.o $(BUILD_DIR)/oci/ref.o $(CJSON_OBJ) | $(BUILD_DIR)
203239
@echo " LD $@"
204240
$(Q)$(CC) $(CFLAGS) -o $@ $^
205241

@@ -217,6 +253,98 @@ $(BUILD_DIR)/test-oci-inspect: $(BUILD_DIR)/test-oci-inspect.o $(BUILD_DIR)/oci/
217253
@echo " LD $@"
218254
$(Q)$(CC) $(CFLAGS) -o $@ $^
219255

256+
## Build the OCI tar reader unit test (native macOS, no HVF). Pure C; the
257+
## test constructs ustar / GNU long-name streams in memory and drives them
258+
## through the reader via a callback that exercises short-read chunking.
259+
$(BUILD_DIR)/test-oci-tar: $(BUILD_DIR)/test-oci-tar.o $(BUILD_DIR)/oci/tar.o | $(BUILD_DIR)
260+
@echo " LD $@"
261+
$(Q)$(CC) $(CFLAGS) -o $@ $^
262+
263+
## Build the OCI runspec unit test (native macOS, no HVF). Pure-data
264+
## merge of image-config runtime block + CLI overrides; the test feeds
265+
## oci_image_runtime_t literals directly through oci_runspec_build with
266+
## no filesystem or libcurl dependency.
267+
$(BUILD_DIR)/test-oci-runspec: $(BUILD_DIR)/test-oci-runspec.o $(BUILD_DIR)/oci/runspec.o | $(BUILD_DIR)
268+
@echo " LD $@"
269+
$(Q)$(CC) $(CFLAGS) -o $@ $^
270+
271+
## Build the OCI path-resolve unit test (native macOS, no HVF). Touches
272+
## the host filesystem to build a small fake sysroot tree and drives
273+
## oci_path_resolve through realpath / stat / symlink-follow scenarios.
274+
## Pure C; no libcurl, no zstd, no HVF.
275+
$(BUILD_DIR)/test-oci-path-resolve: $(BUILD_DIR)/test-oci-path-resolve.o $(BUILD_DIR)/oci/path-resolve.o | $(BUILD_DIR)
276+
@echo " LD $@"
277+
$(Q)$(CC) $(CFLAGS) -o $@ $^
278+
279+
## Build the OCI run orchestrator unit test (native macOS, no HVF). Links
280+
## the same OCI graph the unpack test pulls in, plus oci/run.o,
281+
## oci/runspec.o, and oci/path-resolve.o. Does NOT link core/launch.o:
282+
## the test ships an in-file elfuse_launch stub that aborts when called,
283+
## and every case installs a launch hook via oci_run_set_launch_for_testing
284+
## before invoking oci_run, so the real VM bring-up never runs from a test.
285+
$(BUILD_DIR)/test-oci-run: $(BUILD_DIR)/test-oci-run.o $(BUILD_DIR)/oci/run.o $(BUILD_DIR)/oci/runspec.o $(BUILD_DIR)/oci/path-resolve.o $(BUILD_DIR)/oci/unpack.o $(BUILD_DIR)/oci/volume.o $(BUILD_DIR)/oci/clone-rootfs.o $(BUILD_DIR)/oci/layer-apply.o $(BUILD_DIR)/oci/layer-meta.o $(BUILD_DIR)/oci/decompress.o $(BUILD_DIR)/oci/tar.o $(BUILD_DIR)/oci/store.o $(BUILD_DIR)/oci/blob-store.o $(BUILD_DIR)/oci/digest.o $(BUILD_DIR)/oci/manifest.o $(BUILD_DIR)/oci/media-type.o $(BUILD_DIR)/oci/ref.o $(BUILD_DIR)/core/sysroot.o $(BUILD_DIR)/debug/log.o $(CJSON_OBJ) $(ZSTD_OBJS) | $(BUILD_DIR)
286+
@echo " LD $@"
287+
$(Q)$(CC) $(CFLAGS) -o $@ $^ -lz
288+
289+
## Build the OCI fixture builder (Phase 3 compat tests). Standalone tool
290+
## that synthesises a complete OCI store from uncompressed-tar layers
291+
## plus image-config flags. Used by tests/test-oci-compat.sh and
292+
## available standalone for one-off "shape an image from local files"
293+
## experiments.
294+
$(BUILD_DIR)/oci-fixture-builder: $(BUILD_DIR)/lib/oci-fixture-builder.o $(BUILD_DIR)/oci/store.o $(BUILD_DIR)/oci/blob-store.o $(BUILD_DIR)/oci/digest.o $(BUILD_DIR)/oci/ref.o $(CJSON_OBJ) | $(BUILD_DIR)
295+
@echo " LD $@"
296+
$(Q)$(CC) $(CFLAGS) -o $@ $^
297+
298+
## decompress.c is the only translation unit in elfuse that includes
299+
## externals/zstd/lib/zstd.h. Attach the zstd include path as a target-
300+
## specific CFLAG so the rest of the codebase never sees zstd headers.
301+
$(BUILD_DIR)/oci/decompress.o: CFLAGS += -I$(ZSTD_DIR)/lib
302+
303+
## Build the OCI sidecar metadata unit test (native macOS, no HVF). Pure
304+
## C; links against cJSON for the JSON round-trip plus the layer-meta
305+
## translation unit.
306+
$(BUILD_DIR)/test-oci-meta: $(BUILD_DIR)/test-oci-meta.o $(BUILD_DIR)/oci/layer-meta.o $(CJSON_OBJ) | $(BUILD_DIR)
307+
@echo " LD $@"
308+
$(Q)$(CC) $(CFLAGS) -o $@ $^
309+
310+
## Build the OCI layer applier unit test (native macOS, no HVF). Builds
311+
## tar payloads in memory, drives them through oci_layer_apply into a
312+
## tmp tree, and verifies filesystem state via lstat/readlink.
313+
$(BUILD_DIR)/test-oci-layer-apply: $(BUILD_DIR)/test-oci-layer-apply.o $(BUILD_DIR)/oci/layer-apply.o $(BUILD_DIR)/oci/layer-meta.o $(BUILD_DIR)/oci/tar.o $(CJSON_OBJ) | $(BUILD_DIR)
314+
@echo " LD $@"
315+
$(Q)$(CC) $(CFLAGS) -o $@ $^
316+
317+
## Build the OCI volume bootstrap unit test (native macOS, no HVF).
318+
## Default-volume test is gated behind OCI_VOLUME_TEST=1 because it
319+
## costs ~150 ms of hdiutil orchestration on first run. Links
320+
## src/core/sysroot.o for the hdiutil wrappers PR #33 introduced.
321+
$(BUILD_DIR)/test-oci-volume: $(BUILD_DIR)/test-oci-volume.o $(BUILD_DIR)/oci/volume.o $(BUILD_DIR)/core/sysroot.o $(BUILD_DIR)/debug/log.o | $(BUILD_DIR)
322+
@echo " LD $@"
323+
$(Q)$(CC) $(CFLAGS) -o $@ $^
324+
325+
## Build the OCI clone-rootfs unit test (native macOS, no HVF). The
326+
## test skips itself if clonefile returns ENOTSUP (non-APFS scratch).
327+
$(BUILD_DIR)/test-oci-clone: $(BUILD_DIR)/test-oci-clone.o $(BUILD_DIR)/oci/clone-rootfs.o | $(BUILD_DIR)
328+
@echo " LD $@"
329+
$(Q)$(CC) $(CFLAGS) -o $@ $^
330+
331+
## Build the OCI unpack orchestrator integration smoke (native macOS,
332+
## no HVF). Pulls in the full Phase 2 OCI stack so the dependency
333+
## edges between modules are exercised at link time.
334+
$(BUILD_DIR)/test-oci-unpack: $(BUILD_DIR)/test-oci-unpack.o $(BUILD_DIR)/oci/unpack.o $(BUILD_DIR)/oci/volume.o $(BUILD_DIR)/oci/clone-rootfs.o $(BUILD_DIR)/oci/layer-apply.o $(BUILD_DIR)/oci/layer-meta.o $(BUILD_DIR)/oci/decompress.o $(BUILD_DIR)/oci/tar.o $(BUILD_DIR)/oci/store.o $(BUILD_DIR)/oci/blob-store.o $(BUILD_DIR)/oci/digest.o $(BUILD_DIR)/oci/manifest.o $(BUILD_DIR)/oci/media-type.o $(BUILD_DIR)/oci/ref.o $(BUILD_DIR)/core/sysroot.o $(BUILD_DIR)/debug/log.o $(CJSON_OBJ) $(ZSTD_OBJS) | $(BUILD_DIR)
335+
@echo " LD $@"
336+
$(Q)$(CC) $(CFLAGS) -o $@ $^ -lz
337+
338+
## Build the OCI decompression dispatch unit test (native macOS, no HVF).
339+
## Links zstd objects + system zlib so gzip and zstd payloads both round-
340+
## trip through oci_stream_t. The gzip fixture is generated at test time
341+
## via zlib; the zstd fixture is an embedded byte array because the
342+
## vendored libzstd is decode-only.
343+
$(BUILD_DIR)/test-oci-decompress.o: CFLAGS += -I$(ZSTD_DIR)/lib
344+
$(BUILD_DIR)/test-oci-decompress: $(BUILD_DIR)/test-oci-decompress.o $(BUILD_DIR)/oci/decompress.o $(ZSTD_OBJS) | $(BUILD_DIR)
345+
@echo " LD $@"
346+
$(Q)$(CC) $(CFLAGS) -o $@ $^ -lz
347+
220348
# Guest test binaries (cross-compiled, aarch64-linux)
221349
# Only used when GUEST_TEST_BINARIES is not set.
222350

docs/usage.md

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,93 @@ and memory access, and per-thread inspection. Implementation details, including
9999
the snapshot protocol used to keep Hypervisor.framework register access on the
100100
owning thread, are documented in [internals.md](internals.md).
101101

102+
## Running OCI Images (`elfuse oci run`)
103+
104+
Phase 3 adds a direct-execution path for pulled OCI images:
105+
106+
```sh
107+
elfuse oci run [OPTIONS] IMAGE [ARG...]
108+
```
109+
110+
The subcommand reads the image's runtime block (Entrypoint, Cmd, Env,
111+
WorkingDir, User) and folds in any CLI overrides, then unpacks the image
112+
into the local APFS sysroot volume, clones a per-run rootfs via APFS
113+
`clonefile(2)`, resolves argv[0] against PATH inside the rootfs, and
114+
hands off to the same VM bring-up the legacy positional-ELF `elfuse`
115+
entry uses.
116+
117+
The image must already be pulled. `oci run` does not auto-pull on miss.
118+
The usual workflow is:
119+
120+
```sh
121+
elfuse oci pull alpine:3
122+
elfuse oci run alpine:3 /bin/sh -c 'echo hello from inside'
123+
```
124+
125+
### Options
126+
127+
| Option | Meaning |
128+
|--------|---------|
129+
| `--store DIR` | Override the local store root |
130+
| `--volume DIR` | Override the APFS sysroot volume mount point |
131+
| `--entrypoint PROG` | Replace the image Entrypoint with `PROG` |
132+
| `-e KEY=VAL`, `--env KEY=VAL` | Set or replace one env var (repeatable) |
133+
| `-e KEY`, `--env KEY` | Import `KEY` from the host environ (repeatable) |
134+
| `-w DIR`, `--workdir DIR` | Override image WorkingDir |
135+
| `-u UID[:GID]`, `--user UID[:GID]` | Override image User (numeric only) |
136+
| `--keep` | Keep the per-run cloned rootfs after exit |
137+
| `--name NAME` | Reserved: deterministic clone-dir suffix (ignored today) |
138+
139+
### Argv override matrix
140+
141+
| Image Entrypoint | Image Cmd | CLI ARGV | `--entrypoint` | Result argv |
142+
|--|--|--|--|--|
143+
| set | set | none | none | Entrypoint ++ Cmd |
144+
| set | set | provided | none | Entrypoint ++ CLI ARGV (Cmd dropped) |
145+
| set | none | provided | none | Entrypoint ++ CLI ARGV |
146+
| none | set | none | none | Cmd |
147+
| none | set | provided | none | CLI ARGV (Cmd dropped) |
148+
| set | set | optional | provided | [`--entrypoint`] ++ CLI ARGV |
149+
| none | none | provided | none | CLI ARGV |
150+
| none | none | none | none | `EINVAL` "image has no entrypoint or cmd; pass one on the CLI" |
151+
152+
### Env merge policy
153+
154+
The merged guest env is built in this order:
155+
156+
1. Image `Env` (verbatim, in spec order)
157+
2. Each CLI `-e KEY=VAL` set-or-replaces by key
158+
3. Each CLI `-e KEY` (no `=`) imports the host's value when present, otherwise drops silently
159+
4. `TERM` auto-imported from the host iff the merged env has no `TERM`
160+
5. `PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin` injected iff the merged env has no `PATH`
161+
6. `container=elfuse` injected unconditionally so systemd-style sandbox detection works
162+
163+
CLI `-e DYLD_*=...` overrides are hard-rejected with `EINVAL`: `DYLD_*` is a
164+
macOS-only loader contract with no meaning inside an aarch64-linux guest.
165+
Image-provided `DYLD_*` entries pass through (the guest ignores them).
166+
167+
### User and WorkingDir
168+
169+
`User` accepts numeric `UID` or `UID:GID` only. Symbolic users (`User
170+
nginx`) are rejected with a deterministic Phase 4 pointer message;
171+
static `/etc/passwd` parsing waits for Phase 4 along with the rest of
172+
the NSS resolution work. `--user UID` alone defaults GID to the same
173+
value.
174+
175+
`WorkingDir` must be absolute and free of `..` segments. If neither the
176+
image nor the CLI sets it, the guest starts in `/`. The directory is
177+
materialized under the cloned rootfs (`mkdir -p`, mode 0755, best-
178+
effort chown to the resolved uid:gid when `--user` or image User
179+
selects credentials).
180+
181+
### Scope guardrails
182+
183+
- Symbolic `User` -> Phase 4 (NSS / static `/etc/passwd` resolution)
184+
- `/etc/resolv.conf`, `/etc/hosts`, `/dev/*`, `/proc/*` synthesis -> Phase 4
185+
- Auto-pull on `run` miss -> never; `elfuse oci pull` must run first
186+
- Network policy, `docker run -p`-style port mapping -> later phases
187+
- Live `docker exec`-style attach -> never
188+
102189
## Guest Compatibility Model
103190

104191
`elfuse` is designed for Linux user-space workloads, not for booting a Linux

externals/zstd/LICENSE

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
BSD License
2+
3+
For Zstandard software
4+
5+
Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved.
6+
7+
Redistribution and use in source and binary forms, with or without modification,
8+
are permitted provided that the following conditions are met:
9+
10+
* Redistributions of source code must retain the above copyright notice, this
11+
list of conditions and the following disclaimer.
12+
13+
* Redistributions in binary form must reproduce the above copyright notice,
14+
this list of conditions and the following disclaimer in the documentation
15+
and/or other materials provided with the distribution.
16+
17+
* Neither the name Facebook, nor Meta, nor the names of its contributors may
18+
be used to endorse or promote products derived from this software without
19+
specific prior written permission.
20+
21+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
22+
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
23+
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
24+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
25+
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
26+
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
27+
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
28+
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29+
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
30+
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

externals/zstd/VENDORING.md

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
# Vendored zstd (decode-only)
2+
3+
This directory contains a decode-only subset of [zstd](https://github.com/facebook/zstd),
4+
the streaming compression library from Facebook. Phase 2 needs zstd to
5+
decompress OCI image layers that ship with `application/vnd.oci.image.layer.v1.tar+zstd`
6+
or `application/vnd.docker.image.rootfs.diff.tar.zstd` media types.
7+
8+
Licensed under BSD-3-Clause (see `LICENSE`).
9+
10+
## Why vendored, decode-only
11+
12+
`oci-roadmap.md` Q9 commits the OCI work to hand-rolled C: no Go, no Rust,
13+
no `cargo` / `go` in the build matrix. zstd is the only OCI-spec layer
14+
compression beyond gzip that has wide registry support, and the upstream
15+
library cleanly separates decoder-only from the full encoder. Phase 2 only
16+
reads layers, so the encoder, dictionary builder, and legacy v01-v06
17+
support are all excluded.
18+
19+
Resulting footprint:
20+
21+
- `lib/{zstd,zstd_errors}.h`
22+
- `lib/common/*.{c,h}` (allocator, error tables, FSE/Huff decoders,
23+
threading stubs, xxhash, portability shims)
24+
- `lib/decompress/*.{c,h}` (the streaming decode state machine)
25+
26+
Compression, dictBuilder, deprecated, and legacy paths are NOT vendored.
27+
Do not call `ZSTD_compress*`, `ZDICT_*`, or any `ZSTD_v0X_*` symbol.
28+
29+
## Version
30+
31+
Pinned to upstream tag `v1.5.6` (2024-03-30). Fetched with:
32+
33+
```
34+
curl -fsSL -o zstd-v1.5.6.tar.gz \
35+
https://github.com/facebook/zstd/archive/refs/tags/v1.5.6.tar.gz
36+
tar -xzf zstd-v1.5.6.tar.gz
37+
SRC=zstd-1.5.6
38+
cp $SRC/LICENSE externals/zstd/LICENSE
39+
cp $SRC/lib/zstd.h $SRC/lib/zstd_errors.h externals/zstd/lib/
40+
cp $SRC/lib/common/*.[ch] externals/zstd/lib/common/
41+
cp $SRC/lib/decompress/*.[ch] externals/zstd/lib/decompress/
42+
```
43+
44+
Note: `lib/decompress/huf_decompress_amd64.S` is intentionally NOT
45+
copied. The build sets `-DZSTD_DISABLE_ASM=1` so `huf_decompress.c`
46+
references no AMD64 assembly symbols; the elfuse host is Apple Silicon
47+
in any case.
48+
49+
## Local modifications
50+
51+
None. The files are byte-identical to the upstream tag so future
52+
security updates can be applied by re-running the curl + cp commands
53+
above.
54+
55+
## Build integration
56+
57+
The Makefile compiles each `externals/zstd/lib/**/*.c` translation unit
58+
with project warning flags relaxed (`-Wno-pedantic -Wno-shadow
59+
-Wno-strict-prototypes -Wno-missing-prototypes`) and with the
60+
configuration macros:
61+
62+
```
63+
-DZSTD_DISABLE_ASM=1
64+
-DZSTD_LEGACY_SUPPORT=0
65+
-DZSTD_MULTITHREAD=0
66+
-DZSTDLIB_VISIBILITY=
67+
```
68+
69+
Only `src/oci/decompress.c` includes `externals/zstd/lib/zstd.h`; the
70+
rest of the codebase never sees zstd headers. The zstd objects are
71+
statically embedded into the `elfuse` binary, so no `-lzstd` link line
72+
is needed.

0 commit comments

Comments
 (0)