diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml new file mode 100644 index 0000000..8ba3595 --- /dev/null +++ b/.github/workflows/CI.yml @@ -0,0 +1,173 @@ +name: CI +on: + push: + branches: + - master + - main + tags: ['*'] + pull_request: + workflow_dispatch: +concurrency: + # Skip intermediate builds: always. + # Cancel intermediate builds: only if it is a pull request build. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} +jobs: + test: + name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} + runs-on: ${{ matrix.os }} + # --------------------------------------------------------------------------- + # Package-image fixture for the depot-dependent tests. + # + # Several tests need a REAL pkgimage fixture (.ji + .so under + # compiled/v.). Locally they scan Base.DEPOT_PATH; in CI we + # provision the exact Piccolissimo BuildImage bundle (its `target/pkgimage/` + # depot) from R2 and point the tests at it via JSD_FIXTURE_DEPOT. Without a + # bundle the tests SKIP (green), so this whole mechanism is best-effort. + # + # Required repo secrets (R2 read access to bucket `harmoniqs-binaries`): + # R2_ACCOUNT_ID -> endpoint https://.r2.cloudflarestorage.com + # R2_ACCESS_KEY_ID -> S3 access key id + # R2_SECRET_ACCESS_KEY -> S3 secret access key + # When R2_ACCESS_KEY_ID is empty/unset (e.g. on forks) HAVE_R2 is 'false' and + # the download step is skipped entirely; the fixture tests then skip. + # + # DEPENDENCY: a published bundle must exist. BuildImage.yml (in the + # Piccolissimo repo) uploads tarballs to + # s3://harmoniqs-binaries/piccolissimo/pkgimage/... + # Until a bundle for the runner's (platform, julia) has been published, + # this step finds nothing, logs a warning, and the fixture tests skip. + # + # PRODUCER-SIDE TODO (separate Piccolissimo change, NOT in this repo): publish + # a STABLE pointer object at + # piccolissimo/pkgimage/fixtures/julia/latest.tar.zst + # for each supported Julia patch so this step does not have to parse the + # manifest. Until that key exists, the manifest.json fallback below is used. + # --------------------------------------------------------------------------- + env: + # Derived solely from a secret's presence so it is safe in `if:` (forks + # have no secrets -> 'false'). Note: secret VALUES are never compared in + # `if:`, only presence, so nothing sensitive leaks into logs. + HAVE_R2: ${{ secrets.R2_ACCESS_KEY_ID != '' }} + R2_BUCKET: harmoniqs-binaries + R2_PREFIX: piccolissimo/pkgimage + strategy: + fail-fast: false + matrix: + version: + - '1.12' + os: + - ubuntu-latest + arch: + - x64 + steps: + - uses: actions/checkout@v4 + - uses: julia-actions/setup-julia@v2 + with: + version: ${{ matrix.version }} + arch: ${{ matrix.arch }} + - uses: julia-actions/cache@v2 + - uses: julia-actions/julia-buildpkg@v1 + + # Download + extract the Piccolissimo pkgimage bundle from R2 and export + # JSD_FIXTURE_DEPOT so the depot-dependent tests RUN instead of skipping. + # Gated on R2 secrets being present; best-effort (never fails the job). + - name: Provision pkgimage fixture from R2 + if: env.HAVE_R2 == 'true' + shell: bash + env: + AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: auto + R2_ENDPOINT: https://${{ secrets.R2_ACCOUNT_ID }}.r2.cloudflarestorage.com + run: | + set -uo pipefail # NOT -e: this step must never fail the job. + + warn() { echo "::warning::pkgimage fixture: $*"; } + + # Tools: awscli for S3-compatible R2 access; unzstd to extract. + if ! command -v aws >/dev/null 2>&1; then + echo "Installing awscli..." + sudo apt-get update -y && sudo apt-get install -y awscli || { + warn "could not install awscli; fixture tests will skip"; exit 0; } + fi + if ! command -v unzstd >/dev/null 2>&1; then + sudo apt-get update -y && sudo apt-get install -y zstd || { + warn "could not install zstd; fixture tests will skip"; exit 0; } + fi + + s3() { aws --endpoint-url "$R2_ENDPOINT" s3 "$@"; } + s3api() { aws --endpoint-url "$R2_ENDPOINT" s3api "$@"; } + + # Identify the runner's Julia patch + platform to match a build. + JULIA_PATCH="$(julia -e 'print(VERSION)')" + # Base arch-os-libc triple (e.g. x86_64-linux-gnu) — the simple platform + # tag BuildImage names tarballs with. This deliberately drops the long + # ABI/julia_version suffix from BinaryPlatforms.triplet so it matches the + # `platform` field the producer writes into manifest.json / the filename. + # NOTE: the producer (Piccolissimo BuildImage) OWNS this string format; + # keep this in sync with how it names tarballs / manifest `platform`. + PLATFORM="$(julia -e ' + import Base.BinaryPlatforms: HostPlatform, arch, os, libc + p = HostPlatform() + lc = something(libc(p), "") + lc == "glibc" && (lc = "gnu") # normalize to BinaryBuilder convention + print(join(filter(!isempty, [arch(p), os(p), lc]), "-")) + ')" + echo "Looking for fixture: platform=$PLATFORM julia=$JULIA_PATCH" + + TARBALL="$RUNNER_TEMP/jsd-fixture.tar.zst" + + # 1) Preferred: a STABLE producer-published pointer key. + STABLE_KEY="$R2_PREFIX/fixtures/julia${JULIA_PATCH}/latest.tar.zst" + echo "Trying stable key s3://$R2_BUCKET/$STABLE_KEY ..." + if s3 cp "s3://$R2_BUCKET/$STABLE_KEY" "$TARBALL" >/dev/null 2>&1; then + echo "Got stable fixture." + else + # 2) Fallback: read manifest.json and pick the newest matching build. + echo "Stable key not found; falling back to manifest.json" + MANIFEST="$RUNNER_TEMP/pkgimage-manifest.json" + if ! s3 cp "s3://$R2_BUCKET/$R2_PREFIX/manifest.json" "$MANIFEST" >/dev/null 2>&1; then + warn "no stable fixture and no manifest.json (has BuildImage run?); skipping" + exit 0 + fi + # manifest.json is expected to be a list of build records with at least + # { "platform": ..., "julia_version": ..., "key": ..., and a sortable + # recency field "created_at" (ISO-8601) or "build" }. Pick newest match + # using jq (installed if missing). + command -v jq >/dev/null 2>&1 || sudo apt-get install -y jq >/dev/null 2>&1 || true + KEY="" + if command -v jq >/dev/null 2>&1; then + KEY="$(jq -r --arg plat "$PLATFORM" --arg jv "$JULIA_PATCH" ' + [ .[] | select(.platform == $plat and .julia_version == $jv) ] + | sort_by(.created_at // .build // "") + | last + | .key // empty + ' "$MANIFEST" 2>/dev/null || true)" + else + warn "jq unavailable; cannot parse manifest.json; skipping" + fi + if [ -z "${KEY:-}" ]; then + warn "manifest.json has no build matching ($PLATFORM, julia$JULIA_PATCH); skipping" + exit 0 + fi + echo "Selected manifest key: $KEY" + if ! s3 cp "s3://$R2_BUCKET/$KEY" "$TARBALL" >/dev/null 2>&1; then + warn "failed to download $KEY; skipping" + exit 0 + fi + fi + + # Extract: the tarball's top level IS the depot contents + # (compiled/v./...), so the extract dir becomes the depot. + DEPOT="$RUNNER_TEMP/jsd-fixture" + mkdir -p "$DEPOT" + if ! tar --use-compress-program=unzstd -xf "$TARBALL" -C "$DEPOT"; then + warn "failed to extract bundle; skipping" + exit 0 + fi + echo "JSD_FIXTURE_DEPOT=$DEPOT" >> "$GITHUB_ENV" + echo "Provisioned fixture depot at $DEPOT:" + ls -R "$DEPOT" | head -n 40 || true + + - uses: julia-actions/julia-runtest@v1 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a92a602 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +/Manifest.toml +/csrc/build/ +/docs/build/ +*.o +*.cov diff --git a/Project.toml b/Project.toml index 37ee771..41ef762 100644 --- a/Project.toml +++ b/Project.toml @@ -1,5 +1,5 @@ name = "JuliaStaticData" -uuid = "a1b2c3d4-e5f6-7890-abcd-ef1234567890" +uuid = "72ac74b6-0a8a-4afc-a67b-32735e8af5c8" authors = ["JuliaStaticData contributors"] version = "0.1.0" @@ -14,7 +14,7 @@ julia = "1.12" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" -TestItemRunner = "f8b46487-2199-4571-9461-7a1040e0dea8" +TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a" TestItems = "1c621080-faea-4a02-84b6-bbd5e436b8fe" [targets] diff --git a/csrc/Makefile b/csrc/Makefile new file mode 100644 index 0000000..cb1c698 --- /dev/null +++ b/csrc/Makefile @@ -0,0 +1,52 @@ +CC ?= gcc +CFLAGS ?= -Wall -Wextra -O2 -std=c99 +PREFIX ?= /usr/local + +SRCDIR = . +BUILDDIR = build + +HEADER_SRC = $(SRCDIR)/jlstaticdata_header.c +REMAP_SRC = $(SRCDIR)/jlstaticdata_remap.c +CLI_SRC = $(SRCDIR)/jlstaticdata_cli.c +HEADER = $(SRCDIR)/libjlstaticdata.h + +LIB_SRCS = $(HEADER_SRC) $(REMAP_SRC) +LIB_OBJS = $(BUILDDIR)/jlstaticdata_header.o $(BUILDDIR)/jlstaticdata_remap.o +CLI_OBJ = $(BUILDDIR)/jlstaticdata_cli.o + +SHARED_LIB = $(BUILDDIR)/libjlstaticdata.so +STATIC_LIB = $(BUILDDIR)/libjlstaticdata.a +CLI_BIN = $(BUILDDIR)/jlsd-remap + +.PHONY: all clean install lib cli + +all: lib cli + +lib: $(SHARED_LIB) $(STATIC_LIB) + +cli: $(CLI_BIN) + +$(BUILDDIR): + mkdir -p $(BUILDDIR) + +$(BUILDDIR)/%.o: $(SRCDIR)/%.c $(HEADER) | $(BUILDDIR) + $(CC) $(CFLAGS) -fPIC -c -o $@ $< + +$(SHARED_LIB): $(LIB_OBJS) | $(BUILDDIR) + $(CC) $(CFLAGS) -shared -o $@ $(LIB_OBJS) + +$(STATIC_LIB): $(LIB_OBJS) | $(BUILDDIR) + ar rcs $@ $(LIB_OBJS) + +$(CLI_BIN): $(CLI_OBJ) $(STATIC_LIB) | $(BUILDDIR) + $(CC) $(CFLAGS) -o $@ $(CLI_OBJ) $(STATIC_LIB) + +clean: + rm -rf $(BUILDDIR) + +install: all + install -d $(PREFIX)/bin $(PREFIX)/lib $(PREFIX)/include + install -m 755 $(CLI_BIN) $(PREFIX)/bin/ + install -m 644 $(SHARED_LIB) $(PREFIX)/lib/ + install -m 644 $(STATIC_LIB) $(PREFIX)/lib/ + install -m 644 $(HEADER) $(PREFIX)/include/ diff --git a/csrc/jlstaticdata_cli.c b/csrc/jlstaticdata_cli.c new file mode 100644 index 0000000..6be4827 --- /dev/null +++ b/csrc/jlstaticdata_cli.c @@ -0,0 +1,195 @@ +/* + * jlstaticdata_cli.c — CLI for .ji header inspection and build-id remapping. + * + * Usage: + * jlsd-remap --inspect --input + * jlsd-remap --input --output --remap "ModuleName=hi:lo" + */ + +#include "libjlstaticdata.h" +#include +#include +#include + +#define MAX_REMAPS 64 + +static void usage(const char *prog) { + fprintf(stderr, + "Usage: %s [OPTIONS]\n" + "\n" + "OPTIONS:\n" + " --input Input .ji file\n" + " --output Output file (may equal input for in-place)\n" + " --remap Remap spec: \"ModuleName=hi:lo\" (hex uint64)\n" + " May be repeated (max %d)\n" + " --remap-worklist Also remap worklist build_id.lo\n" + " --inspect Print parsed header and exit\n" + " --validate Validate header integrity and exit\n" + " --quiet Suppress informational output\n" + " --version Print version and exit\n" + " --help Show this help\n" + "\n" + "EXAMPLES:\n" + " %s --inspect --input Foo.ji\n" + " %s --input Foo.ji --output Foo_remapped.ji --remap \"Base=deadbeef:01234567\"\n", + prog, MAX_REMAPS, prog, prog); +} + +/** + * Parse a remap spec string: "ModuleName=hi:lo" + * Returns 0 on success. + */ +static int parse_remap_spec(const char *spec, jlsd_remap_entry_t *out) { + /* Find '=' separator */ + const char *eq = strchr(spec, '='); + if (!eq || eq == spec) { + fprintf(stderr, "Error: invalid remap spec (expected ModuleName=hi:lo): %s\n", spec); + return -1; + } + + /* Extract module name */ + size_t name_len = eq - spec; + char *name = (char *)malloc(name_len + 1); + if (!name) return -1; + memcpy(name, spec, name_len); + name[name_len] = '\0'; + out->module_name = name; + + /* Parse "hi:lo" */ + const char *rest = eq + 1; + const char *colon = strchr(rest, ':'); + if (!colon) { + fprintf(stderr, "Error: invalid remap spec (expected hi:lo after =): %s\n", spec); + free(name); + return -1; + } + + char hi_str[32], lo_str[32]; + size_t hi_len = colon - rest; + size_t lo_len = strlen(colon + 1); + if (hi_len >= sizeof(hi_str) || lo_len >= sizeof(lo_str)) { + fprintf(stderr, "Error: hex value too long in remap spec: %s\n", spec); + free(name); + return -1; + } + memcpy(hi_str, rest, hi_len); hi_str[hi_len] = '\0'; + memcpy(lo_str, colon + 1, lo_len); lo_str[lo_len] = '\0'; + + out->target_build_id.hi = strtoull(hi_str, NULL, 16); + out->target_build_id.lo = strtoull(lo_str, NULL, 16); + out->module_uuid.hi = 0; + out->module_uuid.lo = 0; + + return 0; +} + +int main(int argc, char **argv) { + const char *input = NULL; + const char *output = NULL; + int do_inspect = 0; + int do_validate = 0; + int quiet = 0; + int remap_worklist = 0; + jlsd_remap_entry_t remaps[MAX_REMAPS]; + size_t n_remaps = 0; + + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "-h") == 0) { + usage(argv[0]); + return 0; + } else if (strcmp(argv[i], "--version") == 0) { + printf("jlsd-remap 0.1.0 (JI format version %d)\n", JLSD_FORMAT_VERSION); + return 0; + } else if (strcmp(argv[i], "--input") == 0 && i + 1 < argc) { + input = argv[++i]; + } else if (strcmp(argv[i], "--output") == 0 && i + 1 < argc) { + output = argv[++i]; + } else if (strcmp(argv[i], "--inspect") == 0) { + do_inspect = 1; + } else if (strcmp(argv[i], "--validate") == 0) { + do_validate = 1; + } else if (strcmp(argv[i], "--quiet") == 0) { + quiet = 1; + } else if (strcmp(argv[i], "--remap-worklist") == 0) { + remap_worklist = 1; + } else if (strcmp(argv[i], "--remap") == 0 && i + 1 < argc) { + if (n_remaps >= MAX_REMAPS) { + fprintf(stderr, "Error: too many --remap entries (max %d)\n", MAX_REMAPS); + return 1; + } + if (parse_remap_spec(argv[++i], &remaps[n_remaps]) != 0) + return 1; + n_remaps++; + } else { + fprintf(stderr, "Error: unknown option: %s\n", argv[i]); + usage(argv[0]); + return 1; + } + } + + if (!input) { + fprintf(stderr, "Error: --input is required\n"); + usage(argv[0]); + return 1; + } + + /* Inspect mode */ + if (do_inspect) { + jlsd_header_t hdr; + int rc = jlsd_header_parse(input, &hdr); + if (rc != JLSD_OK) { + fprintf(stderr, "Error: failed to parse header (code %d)\n", rc); + return 1; + } + jlsd_header_dump(&hdr, stdout); + jlsd_header_free(&hdr); + return 0; + } + + /* Validate mode */ + if (do_validate) { + jlsd_header_t hdr; + int rc = jlsd_header_parse(input, &hdr); + if (rc != JLSD_OK) { + fprintf(stderr, "INVALID: parse error (code %d)\n", rc); + return 1; + } + rc = jlsd_header_validate(&hdr); + if (rc != JLSD_OK) { + fprintf(stderr, "INVALID: validation error (code %d)\n", rc); + jlsd_header_free(&hdr); + return 1; + } + if (!quiet) + fprintf(stdout, "VALID: %s (format v%u, %zu deps)\n", + input, hdr.format_version, hdr.modlist_count); + jlsd_header_free(&hdr); + return 0; + } + + /* Remap mode */ + if (n_remaps > 0) { + if (!output) output = input; /* in-place by default */ + + uint32_t flags = JLSD_REMAP_MODLIST; + if (remap_worklist) flags |= JLSD_REMAP_WORKLIST; + + int rc = jlsd_remap(input, output, remaps, n_remaps, flags); + + /* Free remap names */ + for (size_t i = 0; i < n_remaps; i++) + free((void *)remaps[i].module_name); + + if (rc != JLSD_OK) { + fprintf(stderr, "Error: remap failed (code %d)\n", rc); + return 1; + } + if (!quiet) + fprintf(stdout, "Remapped %zu entries in %s -> %s\n", n_remaps, input, output); + return 0; + } + + fprintf(stderr, "Error: no action specified. Use --inspect, --validate, or --remap.\n"); + usage(argv[0]); + return 1; +} diff --git a/csrc/jlstaticdata_header.c b/csrc/jlstaticdata_header.c new file mode 100644 index 0000000..dfb3547 --- /dev/null +++ b/csrc/jlstaticdata_header.c @@ -0,0 +1,275 @@ +/* + * jlstaticdata_header.c — .ji header parser for Layer 1. + * + * Parses the binary format produced by: + * write_header() (staticdata_utils.c:505) + * jl_write_header_for_incremental() (staticdata.c:3465) + * write_worklist_for_header() (staticdata_utils.c:531) + * write_dependency_list() (staticdata_utils.c:563) + * write_mod_list() (staticdata_utils.c:409) + */ + +#include "libjlstaticdata.h" +#include +#include +#include + +/* ── Internal I/O helpers ────────────────────────────────── */ + +static int read_uint8(FILE *f, uint8_t *out) { + return fread(out, 1, 1, f) == 1 ? 0 : -1; +} + +static int read_uint16(FILE *f, uint16_t *out) { + return fread(out, 2, 1, f) == 1 ? 0 : -1; +} + +static int read_int32(FILE *f, int32_t *out) { + return fread(out, 4, 1, f) == 1 ? 0 : -1; +} + +static int read_uint64(FILE *f, uint64_t *out) { + return fread(out, 8, 1, f) == 1 ? 0 : -1; +} + +static int read_int64(FILE *f, int64_t *out) { + return fread(out, 8, 1, f) == 1 ? 0 : -1; +} + +/** + * Read a null-terminated C string from a FILE*. + * Returns a heap-allocated copy. Caller must free(). + */ +static char *read_cstring(FILE *f) { + size_t cap = 64, len = 0; + char *buf = (char *)malloc(cap); + if (!buf) return NULL; + + int ch; + while ((ch = fgetc(f)) != EOF && ch != '\0') { + if (len + 1 >= cap) { + cap *= 2; + char *tmp = (char *)realloc(buf, cap); + if (!tmp) { free(buf); return NULL; } + buf = tmp; + } + buf[len++] = (char)ch; + } + buf[len] = '\0'; + return buf; +} + +/** + * Read a module list in write_worklist_for_header or write_mod_list format. + * + * Format per entry: + * int32 name_len + * char[] name (name_len bytes, NOT null-terminated) + * uint64 uuid_hi + * uint64 uuid_lo + * [uint64 build_id_hi] (only if has_buildid_hi) + * uint64 build_id_lo + * Terminated by int32(0). + * + * @param f Open file at the start of the list. + * @param has_buildid_hi Whether each entry includes build_id.hi. + * @param out_entries Output array (heap-allocated). Caller must free. + * @param out_count Number of entries read. + * @return JLSD_OK or error. + */ +static int read_module_list(FILE *f, int has_buildid_hi, + jlsd_module_entry_t **out_entries, size_t *out_count) { + size_t cap = 8, count = 0; + jlsd_module_entry_t *entries = (jlsd_module_entry_t *)calloc(cap, sizeof(jlsd_module_entry_t)); + if (!entries) return JLSD_ERR_ALLOC; + + while (1) { + int32_t name_len; + if (read_int32(f, &name_len) != 0) goto err_io; + if (name_len == 0) break; /* terminator */ + + if (count >= cap) { + cap *= 2; + jlsd_module_entry_t *tmp = (jlsd_module_entry_t *)realloc(entries, cap * sizeof(jlsd_module_entry_t)); + if (!tmp) goto err_alloc; + entries = tmp; + } + + jlsd_module_entry_t *e = &entries[count]; + memset(e, 0, sizeof(*e)); + + /* Name */ + e->name = (char *)malloc(name_len + 1); + if (!e->name) goto err_alloc; + if (fread(e->name, 1, name_len, f) != (size_t)name_len) goto err_io; + e->name[name_len] = '\0'; + + /* UUID */ + if (read_uint64(f, &e->uuid.hi) != 0) goto err_io; + if (read_uint64(f, &e->uuid.lo) != 0) goto err_io; + + /* Build ID */ + if (has_buildid_hi) { + e->file_offset_hi = (int64_t)ftell(f); + if (read_uint64(f, &e->build_id.hi) != 0) goto err_io; + } else { + e->file_offset_hi = -1; + e->build_id.hi = 0; + } + e->file_offset_lo = (int64_t)ftell(f); + if (read_uint64(f, &e->build_id.lo) != 0) goto err_io; + + count++; + } + + *out_entries = entries; + *out_count = count; + return JLSD_OK; + +err_alloc: + for (size_t i = 0; i < count; i++) free(entries[i].name); + free(entries); + return JLSD_ERR_ALLOC; + +err_io: + for (size_t i = 0; i < count; i++) free(entries[i].name); + free(entries); + return JLSD_ERR_IO; +} + +/* ── Public API ──────────────────────────────────────────── */ + +int jlsd_header_parse_file(FILE *f, jlsd_header_t *out) { + memset(out, 0, sizeof(*out)); + + /* Section 0: Base header (write_header, staticdata_utils.c:505) */ + char magic[JLSD_MAGIC_LEN]; + if (fread(magic, 1, JLSD_MAGIC_LEN, f) != JLSD_MAGIC_LEN) + return JLSD_ERR_IO; + if (memcmp(magic, JLSD_MAGIC, JLSD_MAGIC_LEN) != 0) + return JLSD_ERR_BAD_MAGIC; + + if (read_uint16(f, &out->format_version) != 0) return JLSD_ERR_IO; + + uint16_t bom; + if (read_uint16(f, &bom) != 0) return JLSD_ERR_IO; + if (bom != JLSD_BOM) return JLSD_ERR_BAD_BOM; + + if (read_uint8(f, &out->pointer_size) != 0) return JLSD_ERR_IO; + + out->build_uname = read_cstring(f); if (!out->build_uname) return JLSD_ERR_IO; + out->build_arch = read_cstring(f); if (!out->build_arch) return JLSD_ERR_IO; + out->julia_version = read_cstring(f); if (!out->julia_version) return JLSD_ERR_IO; + out->git_branch = read_cstring(f); if (!out->git_branch) return JLSD_ERR_IO; + out->git_commit = read_cstring(f); if (!out->git_commit) return JLSD_ERR_IO; + + if (read_uint8(f, &out->pkgimage) != 0) return JLSD_ERR_IO; + if (read_uint64(f, &out->checksum) != 0) return JLSD_ERR_IO; + if (read_int64(f, &out->data_start) != 0) return JLSD_ERR_IO; + if (read_int64(f, &out->data_end) != 0) return JLSD_ERR_IO; + + /* Section 1: Cache flags */ + if (read_uint8(f, &out->cache_flags) != 0) return JLSD_ERR_IO; + + if (out->pkgimage == 0) { + /* .ji format: worklist + deplist + modlist */ + + /* Section 2: Worklist */ + int rc = read_module_list(f, 0, &out->worklist, &out->worklist_count); + if (rc != JLSD_OK) return rc; + + /* Section 3: Dependency list (skip over) */ + out->deplist_offset = (int64_t)ftell(f); + uint64_t totbytes; + if (read_uint64(f, &totbytes) != 0) return JLSD_ERR_IO; + if (totbytes > 0) { + if (fseek(f, (long)totbytes, SEEK_CUR) != 0) return JLSD_ERR_IO; + } + out->deplist_length = (int64_t)ftell(f) - out->deplist_offset; + } else { + /* .so format: just cache_flags + modlist (no worklist/deplist) */ + out->worklist = NULL; + out->worklist_count = 0; + out->deplist_offset = 0; + out->deplist_length = 0; + } + + /* Section 4 (.ji) or Section 2 (.so): Required modules */ + int rc = read_module_list(f, 1, &out->modlist, &out->modlist_count); + if (rc != JLSD_OK) return rc; + + return JLSD_OK; +} + +int jlsd_header_parse(const char *filepath, jlsd_header_t *out) { + FILE *f = fopen(filepath, "rb"); + if (!f) return JLSD_ERR_IO; + int rc = jlsd_header_parse_file(f, out); + fclose(f); + return rc; +} + +void jlsd_header_free(jlsd_header_t *header) { + free(header->build_uname); + free(header->build_arch); + free(header->julia_version); + free(header->git_branch); + free(header->git_commit); + + if (header->worklist) { + for (size_t i = 0; i < header->worklist_count; i++) + free(header->worklist[i].name); + free(header->worklist); + } + if (header->modlist) { + for (size_t i = 0; i < header->modlist_count; i++) + free(header->modlist[i].name); + free(header->modlist); + } + + memset(header, 0, sizeof(*header)); +} + +int jlsd_header_validate(const jlsd_header_t *header) { + if (header->format_version < JLSD_FORMAT_VERSION) + return JLSD_ERR_BAD_FORMAT; + uint32_t magic_hi = (uint32_t)(header->checksum >> 32); + if (magic_hi != 0 && magic_hi != (uint32_t)JLSD_CHECKSUM_MAGIC) + return JLSD_ERR_BAD_FORMAT; + return JLSD_OK; +} + +void jlsd_header_dump(const jlsd_header_t *header, FILE *out) { + fprintf(out, "Julia Package Image Header\n"); + fprintf(out, "==========================\n"); + fprintf(out, " Format version: %u\n", header->format_version); + fprintf(out, " Pointer size: %u\n", header->pointer_size); + fprintf(out, " Julia version: %s\n", header->julia_version); + fprintf(out, " Git: %s @ %s\n", header->git_branch, header->git_commit); + fprintf(out, " Platform: %s / %s\n", header->build_uname, header->build_arch); + fprintf(out, " Pkgimage: %s\n", header->pkgimage ? "true" : "false"); + fprintf(out, " Cache flags: 0x%02x\n", header->cache_flags); + + uint32_t crc = (uint32_t)(header->checksum & 0xFFFFFFFF); + uint32_t magic_hi = (uint32_t)(header->checksum >> 32); + fprintf(out, " Checksum: 0x%08x (magic: 0x%08x)\n", crc, magic_hi); + fprintf(out, " Data range: %ld .. %ld (%ld bytes)\n", + (long)header->data_start, (long)header->data_end, + (long)(header->data_end - header->data_start)); + + fprintf(out, "\nWorklist (%zu modules):\n", header->worklist_count); + for (size_t i = 0; i < header->worklist_count; i++) { + const jlsd_module_entry_t *e = &header->worklist[i]; + fprintf(out, " %s build_id.lo=0x%016llx\n", + e->name, (unsigned long long)e->build_id.lo); + } + + fprintf(out, "\nRequired modules (%zu dependencies):\n", header->modlist_count); + for (size_t i = 0; i < header->modlist_count; i++) { + const jlsd_module_entry_t *e = &header->modlist[i]; + fprintf(out, " %s build_id=0x%016llx%016llx\n", + e->name, + (unsigned long long)e->build_id.hi, + (unsigned long long)e->build_id.lo); + } +} diff --git a/csrc/jlstaticdata_remap.c b/csrc/jlstaticdata_remap.c new file mode 100644 index 0000000..cb96e6b --- /dev/null +++ b/csrc/jlstaticdata_remap.c @@ -0,0 +1,127 @@ +/* + * jlstaticdata_remap.c — Build-ID remapping for .ji files (Layer 1). + * + * Patches build-id fields in-place using file offsets recorded during parsing. + * The CRC32C checksum is NOT recomputed because only header fields are changed + * (the data blob is untouched). + */ + +#include "libjlstaticdata.h" +#include +#include +#include + +/** + * Copy a file. Returns 0 on success. + */ +static int copy_file(const char *src, const char *dst) { + FILE *in = fopen(src, "rb"); + if (!in) return -1; + FILE *out = fopen(dst, "wb"); + if (!out) { fclose(in); return -1; } + + char buf[8192]; + size_t n; + while ((n = fread(buf, 1, sizeof(buf), in)) > 0) { + if (fwrite(buf, 1, n, out) != n) { + fclose(in); fclose(out); return -1; + } + } + fclose(in); + fclose(out); + return 0; +} + +/** + * Find a matching remap entry for a module entry. + * Matches by name (if remap->module_name != NULL) or by uuid. + * Returns NULL if no match. + */ +static const jlsd_remap_entry_t *find_remap( + const jlsd_module_entry_t *entry, + const jlsd_remap_entry_t *remaps, size_t n_remaps) +{ + for (size_t i = 0; i < n_remaps; i++) { + const jlsd_remap_entry_t *r = &remaps[i]; + if (r->module_name != NULL) { + if (strcmp(entry->name, r->module_name) == 0) + return r; + } else { + if (entry->uuid.hi == r->module_uuid.hi && + entry->uuid.lo == r->module_uuid.lo) + return r; + } + } + return NULL; +} + +/** + * Write a uint64 at a specific file offset. + */ +static int write_uint64_at(FILE *f, int64_t offset, uint64_t value) { + if (fseek(f, (long)offset, SEEK_SET) != 0) return -1; + if (fwrite(&value, 8, 1, f) != 1) return -1; + return 0; +} + +int jlsd_remap(const char *input_path, const char *output_path, + const jlsd_remap_entry_t *remaps, size_t n_remaps, + uint32_t flags) { + if (n_remaps == 0) return JLSD_OK; + + /* Parse header to get file offsets */ + jlsd_header_t hdr; + int rc = jlsd_header_parse(input_path, &hdr); + if (rc != JLSD_OK) return rc; + + /* Copy input to output if different paths */ + if (strcmp(input_path, output_path) != 0) { + if (copy_file(input_path, output_path) != 0) { + jlsd_header_free(&hdr); + return JLSD_ERR_IO; + } + } + + /* Open output for read/write patching */ + FILE *f = fopen(output_path, "r+b"); + if (!f) { + jlsd_header_free(&hdr); + return JLSD_ERR_IO; + } + + /* Patch required modules (Section 4) */ + if (flags & JLSD_REMAP_MODLIST) { + for (size_t i = 0; i < hdr.modlist_count; i++) { + const jlsd_remap_entry_t *r = find_remap(&hdr.modlist[i], remaps, n_remaps); + if (!r) continue; + + if (hdr.modlist[i].file_offset_hi >= 0) { + if (write_uint64_at(f, hdr.modlist[i].file_offset_hi, r->target_build_id.hi) != 0) + goto err_io; + } + if (write_uint64_at(f, hdr.modlist[i].file_offset_lo, r->target_build_id.lo) != 0) + goto err_io; + } + } + + /* Patch worklist (Section 2) */ + if (flags & JLSD_REMAP_WORKLIST) { + for (size_t i = 0; i < hdr.worklist_count; i++) { + const jlsd_remap_entry_t *r = find_remap(&hdr.worklist[i], remaps, n_remaps); + if (!r) continue; + + /* Worklist only stores build_id.lo */ + if (write_uint64_at(f, hdr.worklist[i].file_offset_lo, r->target_build_id.lo) != 0) + goto err_io; + } + } + + fclose(f); + jlsd_header_free(&hdr); + return JLSD_OK; + +err_io: + fclose(f); + jlsd_header_free(&hdr); + return JLSD_ERR_IO; +} diff --git a/csrc/libjlstaticdata.h b/csrc/libjlstaticdata.h new file mode 100644 index 0000000..cdc272d --- /dev/null +++ b/csrc/libjlstaticdata.h @@ -0,0 +1,175 @@ +/* + * libjlstaticdata — Standalone C library for Julia package image manipulation. + * + * Layer 1: No libjulia dependency. Parses and patches .ji file headers. + * + * The .ji header format is defined by: + * write_header() staticdata_utils.c:505 + * jl_write_header_for_incremental() staticdata.c:3465 + * write_worklist_for_header() staticdata_utils.c:531 + * write_mod_list() staticdata_utils.c:409 + */ + +#ifndef LIBJLSTATICDATA_H +#define LIBJLSTATICDATA_H + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* ── Constants ───────────────────────────────────────────── */ + +#define JLSD_MAGIC "\373jli\r\n\032\n" +#define JLSD_MAGIC_LEN 8 +#define JLSD_FORMAT_VERSION 12 +#define JLSD_BOM 0xFEFF +#define JLSD_CHECKSUM_MAGIC 0xfafbfcfdULL + +/* Remap flags */ +#define JLSD_REMAP_MODLIST 0x01 /* Remap dependency build-ids (Section 4) */ +#define JLSD_REMAP_WORKLIST 0x02 /* Remap worklist build-ids (Section 2) */ + +/* Error codes */ +#define JLSD_OK 0 +#define JLSD_ERR_IO -1 +#define JLSD_ERR_BAD_MAGIC -2 +#define JLSD_ERR_BAD_FORMAT -3 +#define JLSD_ERR_BAD_BOM -4 +#define JLSD_ERR_ALLOC -5 +#define JLSD_ERR_NOT_FOUND -6 + +/* ── Types ───────────────────────────────────────────────── */ + +typedef struct { + uint64_t hi; + uint64_t lo; +} jlsd_uuid_t; + +typedef struct { + uint64_t hi; + uint64_t lo; +} jlsd_build_id_t; + +/** + * A module entry from the worklist or required-modules section. + * + * For worklist entries (Section 2), build_id.hi is 0 (not stored in header). + * For required-module entries (Section 4), both halves are present. + */ +typedef struct { + char *name; /* Module name (heap-allocated, null-terminated) */ + jlsd_uuid_t uuid; + jlsd_build_id_t build_id; + int64_t file_offset_hi; /* Byte offset of build_id.hi in file (-1 if N/A) */ + int64_t file_offset_lo; /* Byte offset of build_id.lo in file */ +} jlsd_module_entry_t; + +/** + * Parsed .ji file header. + */ +typedef struct { + /* Base header (write_header, staticdata_utils.c:505) */ + uint16_t format_version; + uint8_t pointer_size; + char *build_uname; /* Heap-allocated */ + char *build_arch; /* Heap-allocated */ + char *julia_version; /* Heap-allocated */ + char *git_branch; /* Heap-allocated */ + char *git_commit; /* Heap-allocated */ + uint8_t pkgimage; + uint64_t checksum; /* Raw value: crc32c | (JLSD_CHECKSUM_MAGIC << 32) */ + int64_t data_start; + int64_t data_end; + + /* Incremental header */ + uint8_t cache_flags; + + /* Worklist (Section 2) — only for .ji (pkgimage=0) */ + size_t worklist_count; + jlsd_module_entry_t *worklist; + + /* Dependency list byte range (Section 3, opaque) */ + int64_t deplist_offset; + int64_t deplist_length; + + /* Required modules (Section 4 for .ji, Section 2 for .so) */ + size_t modlist_count; + jlsd_module_entry_t *modlist; +} jlsd_header_t; + +/** + * A single build-id remap entry. + */ +typedef struct { + const char *module_name; /* Match by name (NULL = match by uuid only) */ + jlsd_uuid_t module_uuid; /* Ignored if module_name != NULL */ + jlsd_build_id_t target_build_id; +} jlsd_remap_entry_t; + +/* ── Parsing ─────────────────────────────────────────────── */ + +/** + * Parse a .ji file header. + * + * @param filepath Path to the .ji file. + * @param out Output header struct. Caller must call jlsd_header_free(). + * @return JLSD_OK on success, negative error code on failure. + */ +int jlsd_header_parse(const char *filepath, jlsd_header_t *out); + +/** + * Parse a .ji header from an already-opened FILE*. + * + * @param f Open file positioned at the start of the JI header. + * @param out Output header struct. + * @return JLSD_OK on success. + */ +int jlsd_header_parse_file(FILE *f, jlsd_header_t *out); + +/** + * Free all heap memory owned by a parsed header. + */ +void jlsd_header_free(jlsd_header_t *header); + +/* ── Inspection ──────────────────────────────────────────── */ + +/** + * Print header contents to a FILE* (human-readable). + */ +void jlsd_header_dump(const jlsd_header_t *header, FILE *out); + +/** + * Validate header integrity (magic, version, checksum magic marker). + * + * @return JLSD_OK if valid. + */ +int jlsd_header_validate(const jlsd_header_t *header); + +/* ── Remapping ───────────────────────────────────────────── */ + +/** + * Remap build-IDs in a .ji file. + * + * Patches build-id fields in the header in-place. Does not modify the data + * blob, so the CRC32C checksum is unaffected (for JLSD_REMAP_MODLIST only). + * + * @param input_path Source .ji file. + * @param output_path Destination file (may equal input_path for in-place). + * @param remaps Array of remap entries. + * @param n_remaps Number of entries. + * @param flags JLSD_REMAP_MODLIST | JLSD_REMAP_WORKLIST. + * @return JLSD_OK on success. + */ +int jlsd_remap(const char *input_path, const char *output_path, + const jlsd_remap_entry_t *remaps, size_t n_remaps, + uint32_t flags); + +#ifdef __cplusplus +} +#endif + +#endif /* LIBJLSTATICDATA_H */ diff --git a/docs/make.jl b/docs/make.jl new file mode 100644 index 0000000..b11e43a --- /dev/null +++ b/docs/make.jl @@ -0,0 +1,20 @@ +using Documenter +using JuliaStaticData + +makedocs(; + modules=[JuliaStaticData], + sitename="JuliaStaticData.jl", + format=Documenter.HTML(; + prettyurls=get(ENV, "CI", "false") == "true", + canonical="https://juliastaticdata.github.io/JuliaStaticData.jl", + ), + pages=[ + "Home" => "index.md", + "User Guide" => "guide.md", + "CLI Reference" => "cli.md", + "API Reference" => "api.md", + "Package Image Format" => "format.md", + "Julia Internals" => "internals.md", + "Source Code Protection" => "protection.md", + ], +) diff --git a/docs/src/api.md b/docs/src/api.md new file mode 100644 index 0000000..0e7cfd3 --- /dev/null +++ b/docs/src/api.md @@ -0,0 +1,48 @@ +# API Reference + +## Types + +```@docs +PkgImageHeader +WorklistEntry +DepModEntry +RemapSpec +VerificationReport +ProtectionReport +``` + +## Header Inspection + +```@docs +parse_header +inspect +``` + +## Build-ID Remapping + +```@docs +remap +remap! +``` + +## Package Image Loading + +```@docs +load_package_image +resolve_dep +resolve_all_deps +``` + +## Transparent Loading Hooks + +```@docs +install_hooks! +uninstall_hooks! +``` + +## Protection Analysis + +```@docs +analyze_protection +strip_image +``` diff --git a/docs/src/cli.md b/docs/src/cli.md new file mode 100644 index 0000000..916cd6e --- /dev/null +++ b/docs/src/cli.md @@ -0,0 +1,122 @@ +# CLI Reference + +JuliaStaticData includes `jlsd-remap`, a standalone C binary for `.ji` file inspection +and build-ID remapping. It has no Julia dependency and can be used in CI pipelines, +Docker builds, or any environment with a C runtime. + +## Building + +```bash +cd JuliaStaticData.jl/csrc +make # builds build/jlsd-remap, build/libjlstaticdata.{so,a} +``` + +Requirements: C99 compiler (gcc, clang). No external dependencies. + +## Commands + +### Inspect + +Print a parsed `.ji` file header: + +```bash +jlsd-remap --inspect --input path/to/Foo.ji +``` + +Output: + +``` +Julia Package Image Header +========================== + Format version: 12 + Pointer size: 8 + Julia version: 1.12.6 + Git: HEAD @ 15346901f0... + Platform: Linux / x86_64 + Pkgimage: false + Cache flags: 0xa3 + Checksum: 0xf82efaed (magic: 0xfafbfcfd) + Data range: 0 .. 0 (0 bytes) + +Worklist (1 modules): + Downloads build_id.lo=0x323bb62f25162e7a + +Required modules (21 dependencies): + Core build_id=0xfdfcfbfaa980b2d4978d542146808636 + Base build_id=0xfdfcfbfaa980b2d40bc5d8dcad34c7de + ... +``` + +### Validate + +Check header integrity (magic bytes, format version, checksum marker): + +```bash +jlsd-remap --validate --input path/to/Foo.ji +# Output: VALID: path/to/Foo.ji (format v12, 21 deps) +``` + +### Remap + +Patch dependency build IDs in a `.ji` file: + +```bash +jlsd-remap \ + --input Foo.ji \ + --output Foo_remapped.ji \ + --remap "Core=deadbeefcafe1234:0123456789abcdef" \ + --remap "Base=aabbccdd11223344:5566778899aabbcc" +``` + +The `--remap` format is `ModuleName=hi:lo` where `hi` and `lo` are hexadecimal +uint64 values (no `0x` prefix). + +Multiple `--remap` entries can be specified (up to 64). + +### Remap Worklist + +To also patch the worklist module's `build_id.lo`: + +```bash +jlsd-remap \ + --input Foo.ji \ + --output Foo_remapped.ji \ + --remap "Foo=0:aaaaaaaaaaaaaaaa" \ + --remap-worklist +``` + +### In-Place Patching + +Set `--output` to the same path as `--input`: + +```bash +jlsd-remap --input Foo.ji --output Foo.ji --remap "Core=aa:bb" +``` + +## Options Summary + +| Option | Description | +|--------|-------------| +| `--input ` | Input `.ji` file (required) | +| `--output ` | Output file; defaults to input (in-place) | +| `--remap ` | Remap spec: `ModuleName=hi:lo` (repeatable) | +| `--remap-worklist` | Also patch worklist `build_id.lo` | +| `--inspect` | Print header and exit | +| `--validate` | Validate header and exit | +| `--quiet` | Suppress informational output | +| `--version` | Print version | +| `--help` | Show help | + +## C Library API + +The CLI is built on `libjlstaticdata`, which can be linked into other C/C++ programs. +See `csrc/libjlstaticdata.h` for the full API: + +- `jlsd_header_parse(path, &hdr)` — Parse a `.ji` header +- `jlsd_header_free(&hdr)` — Free parsed header +- `jlsd_header_dump(&hdr, stdout)` — Print header +- `jlsd_header_validate(&hdr)` — Validate integrity +- `jlsd_remap(input, output, remaps, n, flags)` — Remap build IDs + +Error codes: `JLSD_OK` (0), `JLSD_ERR_IO` (-1), `JLSD_ERR_BAD_MAGIC` (-2), +`JLSD_ERR_BAD_FORMAT` (-3), `JLSD_ERR_BAD_BOM` (-4), `JLSD_ERR_ALLOC` (-5). diff --git a/docs/src/format.md b/docs/src/format.md new file mode 100644 index 0000000..eeb32f0 --- /dev/null +++ b/docs/src/format.md @@ -0,0 +1,224 @@ +# Package Image Format + +This document describes the binary layout of Julia v1.12/v1.13 package image files. +All references are to the Julia v1.12.6 source at tag `v1.12.6`. + +## Overview + +Julia produces two file types when precompiling a package with `use_pkgimages=1` +(the default): + +| File | Extension | Contains | Portable? | +|------|-----------|----------|-----------| +| Cache header | `.ji` | Header metadata, dependency list, source text | Yes (flat binary) | +| Native image | `.so`/`.dylib`/`.dll` | Compiled machine code + embedded data blob | No (ELF/MachO/PE) | + +With `use_pkgimages=0`, only a `.ji` file is produced containing both the header and +the data blob (no native code). + +## .ji File Layout + +### Non-Split Mode (`pkgimages=0`) + +``` + Offset Section Written by + ------ ------- ---------- + 0 Base Header write_header (staticdata_utils.c:505) + 8B magic "\373jli\r\n\032\n" + 2B format_version (uint16, = 12) + 2B BOM (uint16, = 0xFEFF) + 1B pointer_size (uint8, = 8) + var build_uname (null-terminated) + var build_arch (null-terminated) + var julia_version (null-terminated) + var git_branch (null-terminated) + var git_commit (null-terminated) + 1B pkgimage_flag (uint8, = 0) + 8B checksum slot (uint64, backfilled) + 8B data_start (int64, backfilled) + 8B data_end (int64, backfilled) + + Cache Flags staticdata.c:3468 + 1B cache_flags (uint8) + + Worklist write_worklist_for_header + [per module]: (staticdata_utils.c:531) + 4B name_len (int32) + var name (char[]) + 8B uuid.hi (uint64) + 8B uuid.lo (uint64) + 8B build_id.lo (uint64) <- PATCHABLE + 4B terminator (int32 = 0) + + Dependency List write_dependency_list + 8B totbytes (uint64) (staticdata_utils.c:563) + var source records, requires, + preferences (opaque) + + Required Modules write_mod_list + [per dependency]: (staticdata_utils.c:409) + 4B name_len (int32) + var name (char[]) + 8B uuid.hi (uint64) + 8B uuid.lo (uint64) + 8B build_id.hi (uint64) <- PATCHABLE + 8B build_id.lo (uint64) <- PATCHABLE + 4B terminator (int32 = 0) + + Clone Targets staticdata.c:3554 + 4B int32(0) (no targets) + var padding (to JL_CACHE_BYTE_ALIGNMENT) + + ============ DATA BLOB ============ + + Serialized Object Graph jl_save_system_image_to_stream + var sysimg (staticdata.c:3045) + var const_data + var symbols + var relocs + var gvar_record + var fptr_record + + Trailer + var worklist, init_order, + methods, edges, link_ids + + ============ END DATA BLOB ======== + + Source Text write_srctext (precompile.c:24) + [per source file]: + 4B filename_len (int32) + var filename (char[]) + 8B text_len (uint64) + var text (char[]) + + File CRC (appended by compilecache) +``` + +### Split Mode (`pkgimages=1`, Default) + +The `.ji` file is a thin header — no data blob: + +``` + Offset Section + ------ ------- + 0 Base Header (pkgimage_flag = 0, data_start = 0, data_end = 0) + Cache Flags + Worklist + Dependency List + Required Modules + Clone Targets (non-empty: CPU dispatch data) + Source Text + File CRC + .so CRC + (NO data blob) +``` + +## .so File Layout + +The `.so` is a platform-native shared library (ELF on Linux, Mach-O on macOS): + +``` + ELF Header (64 bytes) + Program Headers + Section Headers + + .rodata section: + JI Header (pkgimage_flag = 1) write_header(ff, 1) + (same base header format) (staticdata.c:3530) + data_start, data_end are NON-ZERO + Followed by: cache_flags + mod_list ONLY + (no worklist, no dependency list) + + jl_system_image_data aotcompile.cpp:2073 + Full serialized data blob + (same content as non-split .ji data blob) + + jl_image_pointers aotcompile.cpp:2286 + jl_image_header_t (nshards, nfvars, ngvars) + jl_image_shard_t[] (per-thread tables) + Target dispatch data + + jl_fvar_offsets / jl_gvar_offsets aotcompile.cpp:244 + int32[] offset tables for PIC + + .text section: + Compiled machine code + Multi-versioned clones (SSE, AVX2, AVX-512) + Dispatch thunks (j_*_gfthunk) + + .symtab section (removable with strip -s): + ~1000+ symbol entries + julia_* function names + Compiler intrinsics + + .dynsym section (NOT removable): + ~60 symbols (libjulia ABI) + ijl_apply_generic, ijl_box_int64, etc. +``` + +## Build ID Structure + +Every Julia module carries a 128-bit build ID stored as two 64-bit halves: + +```c +typedef struct { // julia.h:801 + uint64_t hi; + uint64_t lo; +} jl_uuid_t; // also used for build_id +``` + +| Half | Set When | Value | Deterministic? | +|------|----------|-------|----------------| +| `build_id.lo` | Module creation (`module.c:512`) | `bitmix(jl_hrtime() + count, jl_rand())` | No (time + random) | +| `build_id.hi` | Image deserialization (`staticdata.c:4243`) | CRC32C of data blob | Yes (for same blob) | + +The combined 128-bit value is `(UInt128(hi) << 64) \| lo`, accessible via +`Base.module_build_id(m)` (`loading.jl:2532`). + +## Checksum + +The checksum field in the header is a 64-bit value: + +``` +Upper 32 bits: magic marker 0xfafbfcfd +Lower 32 bits: CRC32C of data blob +``` + +Written at `staticdata.c:3577`: +```c +write_uint64(ff, checksum | ((uint64_t)0xfafbfcfd << 32)); +``` + +For split images, the `.ji` gets the checksum backfilled but `data_start` and +`data_end` remain 0 (the data blob is in the `.so`). The `.so` gets all three +fields backfilled. + +## Validation Flow + +On load, Julia validates the header at two levels: + +**C-side** (`staticdata_utils.c:838`): `jl_read_verify_header` checks magic, +format version, BOM, pointer size, platform strings, and Julia version. + +**C-side** (`staticdata_utils.c:797`): `read_verify_mod_list` checks each +dependency's `(name, uuid, build_id.hi, build_id.lo)` against the `depmods` +array with **exact equality** (line 821-822). + +**Julia-side** (`loading.jl:3971`): `stale_cachefile` performs 10+ additional +checks including source file freshness, preferences hash, and concrete dependency +versions. + +Build-ID remapping targets the C-side `write_mod_list`/`read_verify_mod_list` +boundary: by writing the target's build IDs into the header, the C-side validation +passes when the image is loaded against the target installation's modules. + +## Format Versioning + +The format version (`JI_FORMAT_VERSION`) is checked during header validation. +Package images built with a different version are rejected. + +| Julia Version | JI_FORMAT_VERSION | Compatible? | +|---------------|-------------------|-------------| +| 1.12.x | 12 | Yes | +| 1.13.x (rc1) | 12 | Yes | +| master (future 1.14-dev) | 13-14 | No (header changes) | diff --git a/docs/src/guide.md b/docs/src/guide.md new file mode 100644 index 0000000..e6e701d --- /dev/null +++ b/docs/src/guide.md @@ -0,0 +1,242 @@ +# User Guide + +## Inspecting Package Images + +Every `.ji` file has a header recording the package identity, its dependencies, and their +exact build IDs. Use [`parse_header`](@ref) to read it and [`inspect`](@ref) to display it: + +```julia +using JuliaStaticData + +hdr = parse_header("compiled/v1.12/Downloads/abc123.ji") +inspect(hdr) +``` + +Output: + +``` +Julia Package Image Header +========================== + File: compiled/v1.12/Downloads/abc123.ji + Format version: 12 + Pointer size: 8 + Julia version: 1.12.6 + Pkgimage: false + Checksum: 0xf82efaed (magic: 0xfafbfcfd) + +Worklist (1 modules): + Downloads uuid=f43a241f-... build_id.lo=0x323bb62f25162e7a + +Required modules (21 dependencies): + Core uuid=00000000-... build_id=0xfdfcfbfaa980b2d4978d542146808636 + Base uuid=00000000-... build_id=0xfdfcfbfaa980b2d40bc5d8dcad34c7de + ... +``` + +Split images (`.so` files) are also supported — the parser scans for the embedded JI +header with `pkgimage=true`: + +```julia +hdr_so = parse_header("compiled/v1.12/Downloads/abc123.so") +# hdr_so.pkgimage == true +# hdr_so.data_start, hdr_so.data_end point to the embedded data blob +``` + +### Programmatic Access + +The [`PkgImageHeader`](@ref) struct gives you direct access to all header fields: + +```julia +hdr = parse_header("Foo.ji") + +# Package being serialized +for w in hdr.worklist + println(w.name, " uuid=", w.uuid, " build_id.lo=", w.build_id_lo) +end + +# Dependencies this package links against +for dep in hdr.required_modules + full_id = UInt128(dep.build_id_hi) << 64 | dep.build_id_lo + println(dep.name, " build_id=", string(full_id, base=16)) +end +``` + +## Remapping Build IDs + +### The Problem + +Every Julia installation creates sysimage modules (Core, Base, stdlibs) with unique +build IDs derived from `bitmix(jl_hrtime() + count, jl_rand())` (`module.c:512`). +Package images record these exact IDs. Moving a `.ji` file to a different Julia +installation fails because the IDs don't match. + +### The Solution + +[`remap`](@ref) patches the build-ID fields in the `.ji` header, making it compatible +with a target installation: + +```julia +using JuliaStaticData + +# Step 1: Get target build IDs from the target Julia session +# (run on the target machine) +target_core_id = Base.module_build_id(Core) # UInt128 +target_base_id = Base.module_build_id(Base) # UInt128 + +# Step 2: Remap the .ji file +remap("Foo.ji", "Foo_remapped.ji", [ + RemapSpec("Core", nothing, target_core_id), + RemapSpec("Base", nothing, target_base_id), + # ... repeat for all sysimage dependencies +]) +``` + +### Remapping an Entire Dependency Chain + +For a package with many transitive dependencies, build a remap spec for every +module listed in `required_modules`: + +```julia +hdr = parse_header("Foo.ji") + +# Build a remap table from the target session's module build IDs +# target_ids is a Dict{String, UInt128} mapping module names to target build IDs +remaps = [RemapSpec(dep.name, dep.uuid, target_ids[dep.name]) + for dep in hdr.required_modules] + +remap("Foo.ji", "Foo_target.ji", remaps) +``` + +### In-Place Remapping + +Use [`remap!`](@ref) to modify a file in place: + +```julia +remap!("Foo.ji", remaps) +``` + +### Worklist Remapping + +By default, only dependency build IDs (Section 4 of the header) are patched. +To also patch the worklist module's `build_id.lo` (Section 2), pass +`remap_worklist=true`: + +```julia +remap("Foo.ji", "Foo_out.ji", remaps; remap_worklist=true) +``` + +!!! warning + Worklist remapping patches the header but NOT the `build_id.lo` inside the + serialized module struct in the data blob. This creates an inconsistency that + may cause issues with method root block keying. For full consistency, use the + Layer 2 `reserialize()` function (future). + +### Checksum Preservation + +The CRC32C checksum in the header covers only the **data blob**, not the header +fields. Since `remap` only patches header bytes, the checksum is automatically +preserved. No recomputation is needed. + +## Loading Remapped Images + +### Parallel Loading Path + +[`load_package_image`](@ref) provides a loading path parallel to `Base.require`, +calling the same C deserialization functions but with caller-controlled dependency +resolution: + +```julia +using JuliaStaticData + +mod = load_package_image("Foo_remapped.ji") +``` + +The function: +1. Parses the header to discover dependencies +2. Resolves each dependency against loaded modules via [`resolve_all_deps`](@ref) +3. Calls `ccall(:jl_restore_package_image_from_file, ...)` or + `ccall(:jl_restore_incremental, ...)` +4. Registers the module with `Base.register_restored_modules` +5. Returns the loaded `Module` + +### Manual Dependency Resolution + +For finer control, resolve dependencies yourself: + +```julia +hdr = parse_header("Foo_remapped.ji") +deps = [resolve_dep(entry) for entry in hdr.required_modules] +mod = load_package_image("Foo_remapped.ji"; depmods=deps) +``` + +[`resolve_dep`](@ref) searches `Base.loaded_precompiles`, `Base.loaded_modules`, +and the well-known modules (Core, Base, Main) for a module matching the given +name and build ID. + +### Loading Without Registration + +To load a module without registering it with Base (useful for inspection): + +```julia +mod = load_package_image("Foo.ji"; register=false) +# mod is usable but not visible to `using` or `import` +``` + +## Transparent Loading (Hooks) + +!!! warning "Experimental and Fragile" + The hooks mechanism monkey-patches internal Base functions. It works on Julia + 1.12.x but may break on future versions. Prefer the parallel loading path for + production use. + +[`install_hooks!`](@ref) intercepts the standard `using`/`import` path to +transparently handle remapped images: + +```julia +using JuliaStaticData + +install_hooks!(; + remap_table=Dict("Core" => target_core_id, "Base" => target_base_id), + bypass_staleness=true, +) + +using Foo # transparently loads with remapped build IDs + +uninstall_hooks!() # restore original behavior +``` + +## Protection Analysis + +[`analyze_protection`](@ref) scans a package image for information leakage: + +```julia +report = analyze_protection("Foo.ji") +``` + +Output: + +``` +Protection Analysis: Foo.ji +============================================================ + Source text present: YES (HIGH RISK) + Julia IR present: YES (assumed) + Debug info present: YES (assumed) + Metadata present: YES (assumed) + ELF symbols present: no / N/A + +Recommendations: + 1. Strip source text: zero srctextpos section or use strip_image() + 2. Strip IR: rebuild with --strip-ir to remove CodeInfo/inferred code + 3. Strip metadata: rebuild with --strip-metadata to remove file paths/line numbers + 4. Consider --trim=safe to remove unreachable code + 5. Note: method names, type names, and module structure are ALWAYS preserved +``` + +[`strip_image`](@ref) creates a hardened copy with randomized build IDs: + +```julia +strip_image("Foo.ji", "Foo_stripped.ji"; randomize_build_ids=true) +``` + +See [Source Code Protection](@ref) for a thorough analysis of what can and cannot +be hidden in Julia package images. diff --git a/docs/src/index.md b/docs/src/index.md new file mode 100644 index 0000000..c17e09d --- /dev/null +++ b/docs/src/index.md @@ -0,0 +1,79 @@ +# JuliaStaticData.jl + +Low-level toolkit for Julia package image (`.ji`/`.so`) manipulation. + +## What It Does + +Julia precompiles packages into binary cache files (`.ji` and `.so`). These files +contain serialized Julia objects, compiled native code, and metadata that ties them +to the exact Julia installation they were built on. JuliaStaticData gives you +direct access to this machinery: + +- **Inspect** package image headers: see dependencies, build IDs, format metadata +- **Remap** build IDs: make a package image loadable against a different Julia + installation (same version, different build) +- **Load** remapped images programmatically, bypassing the standard staleness checks +- **Analyze** information leakage from package images and apply mitigations + +## Why You Might Need This + +Julia package images are **not portable** across installations. Two copies of Julia +1.12.6 built at different times produce different sysimage build IDs (`module.c:512` +uses `jl_hrtime() + jl_rand()`). Every package compiled on installation A is +incompatible with installation B, even though the Julia version is identical. + +This means: +- CI caches cannot be shared across runners with different Julia builds +- Precompiled packages cannot be deployed without the exact Julia binary +- Custom sysimages are tied to one specific Julia build + +JuliaStaticData breaks this barrier by letting you rewrite the build IDs in package +image headers, making them compatible with any target installation. + +## Architecture + +``` + JuliaStaticData.jl (Julia package) + | parse_header, inspect -- header inspection + | remap, remap! -- build-id patching + | load_package_image -- custom loading + | analyze_protection -- security analysis + | + libjlstaticdata (C library) + | Layer 1: standalone .ji patcher (no libjulia dep) + | Layer 2: libjulia-linked reserializer (future) + | + jlsd-remap (CLI binary) + --inspect, --validate, --remap +``` + +## Quick Start + +```julia +using JuliaStaticData + +# Inspect a package image +inspect("/path/to/compiled/v1.12/Foo/abc123.ji") + +# Remap a dependency to a different build-id +remap("Foo.ji", "Foo_remapped.ji", [ + RemapSpec("Core", nothing, target_core_build_id), + RemapSpec("Base", nothing, target_base_build_id), +]) + +# Load the remapped image +mod = load_package_image("Foo_remapped.ji") +``` + +## Target Julia Versions + +JuliaStaticData supports Julia 1.12.x and 1.13.x (both use `JI_FORMAT_VERSION = 12`). +The core build-ID architecture is preserved on Julia master (future 1.14-dev); line +numbers shift but the format is structurally identical. + +## Contents + +```@contents +Pages = ["guide.md", "cli.md", "api.md", "format.md", "internals.md", "protection.md"] +Depth = 2 +``` diff --git a/docs/src/internals.md b/docs/src/internals.md new file mode 100644 index 0000000..738d145 --- /dev/null +++ b/docs/src/internals.md @@ -0,0 +1,150 @@ +# Julia Internals + +This document explains how Julia's package precompilation and loading infrastructure +works, providing the context needed to understand JuliaStaticData's operations. + +All references are to Julia v1.12.6 source code. + +## The Precompilation Pipeline + +When Julia precompiles a package, three code paths converge: + +### 1. Julia Side: `compilecache` (`loading.jl:3193`) + +The master process: +1. Builds `concrete_deps` from `_concrete_dependencies` + `loaded_modules` (line 3203) + - Core, Base, Main are **excluded** (line 3205) + - Loaded stdlibs and packages are **included** with their `module_build_id` +2. Spawns a worker process via `create_expr_cache` (line 3062) with + `--output-ji` and `--output-o` flags +3. The worker executes `include_package_for_output` (line 3003): + - Sets `Base._concrete_dependencies` (line 3016) + - Loads the package source via `include` (line 3028) + - Calls `ccall(:jl_set_newly_inferred)` to track inference results (line 3026) +4. On worker exit, `jl_write_compiler_output` (precompile.c:98) is called + +### 2. C Side: `jl_create_system_image` (`staticdata.c:3483`) + +The serialization orchestrator: +1. Selects compilation path: + - `jl_precompile_worklist()` for incremental builds (line 3524) + - `jl_precompile()` for fresh sysimages (line 3543) + - `jl_precompile_trimmed()` for `--trim` mode (line 3541) +2. **Disables concurrent Julia execution** (line 3548) — serialization is single-threaded +3. Writes the header via `jl_write_header_for_incremental` (line 3528) +4. Serializes the object graph via `jl_save_system_image_to_stream` (line 3564) +5. Backfills the checksum, `data_start`, `data_end` (lines 3575-3579) + +### 3. C++ Side: `jl_dump_native_impl` (`aotcompile.cpp:1984`) + +The native code emitter (for `.so` files): +1. Receives compiled LLVM modules from `jl_emit_native_impl` (line 768) +2. Partitions across N threads for parallel optimization (line 1242) +3. Writes object files, bitcode, and metadata to an archive +4. Embeds the data blob as `jl_system_image_data` (line 2073) + +## The Loading Pipeline + +### Julia Side: `require` (`loading.jl:2359`) + +``` +require(mod) + -> __require(mod) + -> _require_prelocked(uuidkey) + -> __require_prelocked(pkg, env) + -> _require_search_from_serialized(pkg, sourcepath, build_id) + -> find_all_in_cache_path(pkg) # discover .ji files + -> stale_cachefile(pkg, build_id, ...) # 10+ staleness checks + -> _tryrequire_from_serialized(pkg, path, ocachepath) + -> parse_cache_header(io, path) # read header on Julia side + -> recursively load deps via _tryrequire_from_serialized + -> _include_from_serialized(pkg, path, ocachepath, depmods) + -> ccall(:jl_restore_package_image_from_file, ...) # split image + OR ccall(:jl_restore_incremental, ...) # .ji only + -> register_restored_modules(sv, pkg, path) + -> run __init__ callbacks +``` + +### C Side: `jl_restore_package_image_from_file` (`staticdata.c:4541`) + +1. `jl_dlopen(fname)` — memory-map the `.so` (line 4543) +2. `get_image_buf()` — extract data sections (line 4556) +3. `jl_restore_package_image_from_stream` (line 4392): + - `jl_validate_cache_file` — magic, version, architecture checks (line 4399) + - `read_verify_mod_list` — **exact build-ID match** for every dependency (line 4388) + - CRC32C verification of data blob (line 4425) + - `jl_restore_system_image_from_stream_` — full deserialization (line 4434) + - `jl_copy_roots` — method root block insertion (line 4439) + - `jl_activate_methods` — world counter management (line 4461) + +## The Linkage Mechanism + +This is the key architectural insight that makes build-ID remapping feasible. + +### Indices, Not IDs + +The serialized data blob does **not** contain build IDs. External references use +a two-level indirection through positional indices: + +``` +serialized pointer -> depsidx (index into depmods array) + | + v + buildid_depmods_idxs[depsidx] -> blob_index + | + v + jl_linkage_blobs.items[2*blob_index] -> memory address +``` + +This is implemented in `add_external_linkage` (`staticdata.c:1204`): +- `external_blob_index(v)` finds which loaded image contains the pointer (Eytzinger tree) +- `buildid_depmods_idxs[blob_idx]` maps to the depmods array position +- The depmods index is encoded as a `SysimageLinkage` or `ExternalLinkage` relocation tag + +### Why This Enables Remapping + +Build IDs appear **only in the header** — in `write_mod_list` (dependency verification) +and `write_worklist_for_header` (worklist identity). The data blob uses positional +indices that are reconstructed at load time from the `depmods` array order. + +Therefore, patching the header's build IDs is sufficient to make an image load against +modules with different build IDs, as long as the modules are structurally compatible. + +## Sysimage Module Build IDs + +All sysimage modules (Core, Base, Main, and all stdlibs) share the same `build_id.hi` +because they are part of the same serialized image, sharing the same CRC32C checksum. +Each has a unique `build_id.lo` from its creation timestamp. + +Empirically verified on Julia 1.12.6: +- `Core.build_id.hi = Base.build_id.hi = Main.build_id.hi = LinearAlgebra.build_id.hi` + = `0xfdfcfbfaa980b2d4` +- Each `build_id.lo` is unique (e.g., Core = `0x978d...`, Base = `0x0bc5...`) + +This means a single Julia build produces a consistent set of build IDs. All package +images compiled on that build share the same sysimage dependency IDs. When remapping +for a target installation, you need the target's sysimage build IDs — extractable via: + +```julia +# Run on target machine: +for mod in (Core, Base, Main) + println(nameof(mod), " = ", repr(Base.module_build_id(mod))) +end +``` + +## Method Table Consistency + +The `worklist_key` (`staticdata_utils.c:73`) is `topmod->build_id.lo`. It is used +to key method root blocks during serialization (`staticdata.c:944-960`) and +deserialization (`staticdata_utils.c:770`). + +If you remap a worklist module's `build_id.lo` in the header but not in the data blob, +the root block keys won't match. This is safe for **dependency-only remapping** (the +common case) because only the dependency modules' build IDs are in the `write_mod_list` +section — the worklist module's `build_id.lo` in the data blob is not checked against +the header. + +For full consistency when remapping the worklist module itself, use Layer 2 +(re-serialization), which temporarily patches `mod->build_id.lo` before calling +`jl_create_system_image`, ensuring the data blob, root blocks, and header are all +consistent. diff --git a/docs/src/protection.md b/docs/src/protection.md new file mode 100644 index 0000000..96f774a --- /dev/null +++ b/docs/src/protection.md @@ -0,0 +1,216 @@ +# Source Code Protection + +This document analyzes what information is recoverable from Julia package images and +what mitigations are available. It is intended for developers distributing commercial +or IP-sensitive Julia code. + +## The Short Answer + +Julia package images **cannot fully hide source code**. Method names, type names, type +signatures, and module structure are always preserved regardless of stripping options. +These are required by Julia's runtime for dispatch, error reporting, and the GC. + +What *can* be hidden: source text, Julia IR, file paths, line numbers, variable names, +debug info, unreachable code, and ELF function symbols. + +## Information Inventory + +### What's in a .ji File + +| Information | Default | `--strip-ir` | `--strip-metadata` | Both | +|---|---|---|---|---| +| Source text (.jl files) | Present | Present | Present | Present | +| Julia IR (CodeInfo) | Present | **Removed** | Partial | **Removed** | +| Inferred IR (optimized) | Present | **Removed** | Present | **Removed** | +| Method names | Present | Present | Present | Present | +| Type names and signatures | Present | Present | Present | Present | +| Module hierarchy | Present | Present | Present | Present | +| Source file paths | Present | Present | **Removed** | **Removed** | +| Line numbers | Present | Present | **Removed** | **Removed** | +| Local variable names | Present | Present | **Replaced with "?"** | **Replaced** | +| Debug info | Present | Present | **Removed** | **Removed** | +| Call graph (edges) | Present | **Removed** | Present | **Removed** | +| Binding names | Present | Present | Present | Present | +| Build IDs | Present | Present | Present | Present | + +Source: `staticdata.c:2728-2798` (stripping logic) + +### What's in a .so File (Additional) + +| Information | Default | After `strip -s` | +|---|---|---| +| Compiled machine code | Present | Present | +| ELF symbol table (~1000+ entries) | Present | **Removed** (~60 remain) | +| `julia_*` function name symbols | Present | **Removed** | +| Dynamic symbols (libjulia ABI) | Present (59) | Present (59) | +| Embedded data blob (everything in .ji) | Present | Present | + +Source: `aotcompile.cpp:743` (symbol encoding), `aotcompile.cpp:2073` (data blob embedding) + +## Threat Model + +### Level 1: Julia REPL User (minutes) + +After loading the package, an attacker can use: +- `names(M; all=true)` — all exported and internal binding names +- `methods(f)` — all method signatures with type parameters +- `fieldnames(T)` — struct field names +- `@code_lowered f(args...)` — Julia IR (unless `--strip-ir`) +- `@code_typed f(args...)` — type-inferred IR (unless `--strip-ir`) +- `@which f(args...)` — source file and line (unless `--strip-metadata`) + +**Mitigated by**: `--strip-ir` (removes IR), `--strip-metadata` (removes source locations) + +**Not mitigated**: method names, type names, struct fields, type signatures + +### Level 2: Binary File Inspector (hours) + +Without loading the package: +- `strings Foo.ji` — recovers source text, method names, type names, paths +- `strings Foo.so` — recovers embedded data strings + ELF symbol names +- `nm Foo.so` — lists all function names (`julia_Downloader_3451`, etc.) +- Hex editor — finds header metadata, build IDs, dependency names + +**Mitigated by**: zeroing source text section, `strip -s` on .so + +**Not mitigated**: embedded data blob in .so contains method/type names as binary data + +### Level 3: Julia Internals Expert (days) + +Using tools like JuliaStaticData or custom parsers: +- Parse the data blob to extract all serialized objects +- Reconstruct type definitions, method tables, binding tables +- Extract IR from CodeInfo objects (unless stripped) +- Map the full module dependency graph + +**Mitigated by**: `--strip-ir` (removes CodeInfo), `--trim=safe` (removes unreachable code) + +**Not mitigated**: type definitions, method names, binding names (always serialized) + +### Level 4: Reverse Engineer (weeks) + +Standard binary analysis of the `.so`: +- Disassemble `.text` section +- Reconstruct control flow and algorithms +- Identify library calls via PLT/GOT entries + +**Difficulty**: comparable to reverse engineering any compiled binary (C, Rust, etc.) + +## Mitigation Options + +### Tier 1: Source Text Removal (Highest Impact, Easiest) + +Source text is stored at a known offset in the `.ji` file (`srctextpos` section). +It can be zeroed without affecting package functionality. + +```julia +# The source text section is only used by Base.read_srctext for display +# It is NOT needed for loading or execution +``` + +Impact: eliminates the trivial `strings Foo.ji | grep 'function'` attack. + +### Tier 2: IR Stripping (`--strip-ir`) + +Build with: +```bash +julia --strip-ir --output-ji=Foo.ji --output-o=Foo.o ... +``` + +Removes from `staticdata.c:2754-2776`: +- `jl_method_t.source` → set to `jl_nothing` +- `jl_method_t.roots` → set to NULL +- `jl_code_instance_t.inferred` → set to `jl_nothing` +- `jl_code_instance_t.edges` → set to empty svec + +**Consequence**: `@code_lowered` and `@code_typed` return nothing. Runtime JIT for +methods without compiled native code will fail. + +### Tier 3: Metadata Stripping (`--strip-metadata`) + +Build with: +```bash +julia --strip-metadata --output-ji=Foo.ji --output-o=Foo.o ... +``` + +Removes from `staticdata.c:2778-2798`: +- `jl_method_t.file` → empty symbol +- `jl_method_t.line` → 0 +- `jl_method_t.debuginfo` → `jl_nulldebuginfo` +- `CodeInfo.slotnames` → replaced with "?" + +**Consequence**: stack traces show `"":0` instead of file:line. `@which` returns no path. + +### Tier 4: ELF Symbol Stripping + +```bash +strip -s Foo.so +``` + +Removes ~1,097 of ~1,156 symbols. The 59 dynamic symbols (libjulia ABI) cannot be +removed — they are needed for the dynamic linker. + +### Tier 5: Dead Code Elimination (`--trim=safe`) + +```bash +julia --experimental --trim=safe --output-ji=Foo.ji ... +``` + +Removes all code not provably reachable from annotated entry points: +- Module bindings reduced to modules + `__init__` + runtime essentials +- Method tables rebuilt with only compiled methods +- Backedges and scanned_methods cleared + +**Consequence**: the package can only be used through its declared entry points. +Cannot be used as a general-purpose library. + +**Limitation**: cannot verify dynamic dispatch (`invokelatest`, `Core._apply`, etc.) +See `Compiler/src/verifytrim.jl:235-250` for the unverifiable constructs. + +### Combined: Maximum Protection + +Build with all mitigations: +```bash +julia --experimental --trim=safe --strip-ir --strip-metadata \ + --output-ji=Foo.ji --output-o=Foo.o ... +# Then: +strip -s Foo.so +# Then zero source text in .ji via JuliaStaticData: +julia -e 'using JuliaStaticData; strip_image("Foo.ji", "Foo.ji")' +``` + +**What remains**: method names, type names, type signatures, module structure, +compiled machine code, 59 dynamic ELF symbols. + +## The Fundamental Limitation + +Julia's type system requires self-describing objects. Every type carries its name, +field names, and supertype chain. Every method carries its name and signature. +These are used by: + +- Multiple dispatch (needs type names for method selection) +- Error messages (needs method/type names for reporting) +- The GC (needs type layout for traversal) +- Serialization (needs type identity for consistency) + +**No amount of stripping can remove this information** while keeping the package +functional within Julia. If hiding the API surface is a hard requirement, consider: + +1. **PackageCompiler.jl** with `--trim=safe` — produces a standalone executable + with only entry-point functions visible +2. **C-callable interface** — expose only `@ccallable` functions, keeping Julia + internals in the binary +3. **Service architecture** — run Julia server-side, expose only an API (HTTP, gRPC) + +## Portability vs Security Summary + +| Strategy | Portability | Security | Build-ID Remappable? | +|----------|-------------|----------|---------------------| +| .ji only (pkgimages=0) | High | Low | Yes (trivial) | +| .ji + .so (default) | Medium | Medium | .ji yes, .so difficult | +| .so only (custom loader) | Low | Medium-High | Requires Layer 2 | +| Stripped .so + minimal .ji | Low | Highest achievable | .ji yes, .so requires Layer 2 | + +There is no format that provides both full portability and strong security. This is a +fundamental tension: portable formats must be self-describing, and self-describing +formats are introspectable. diff --git a/src/header.jl b/src/header.jl index a0de95a..591a57b 100644 --- a/src/header.jl +++ b/src/header.jl @@ -63,7 +63,12 @@ end """ parse_header(path::String) -> PkgImageHeader -Parse the header of a `.ji` package image file. Does not read the data blob. +Parse the header of a `.ji` or `.so` package image file. + +For `.ji` files, reads the header directly from the file. +For `.so`/`.dylib`/`.dll` files (split images), loads the shared library via +`Libdl.dlopen`, locates the embedded JI header via `jl_image_pointers`, and +parses it from memory. The parser replicates the binary format from: - `write_header` (staticdata_utils.c:505-523) @@ -72,14 +77,105 @@ The parser replicates the binary format from: - `_parse_cache_header` (base/loading.jl:3430-3491) # Throws -- `ArgumentError` if the file is not a valid `.ji` file +- `ArgumentError` if the file is not a valid Julia package image """ function parse_header(path::String) + if _is_shared_library(path) + return _parse_so_header(path) + end open(path, "r") do io return _parse_header_from_io(io, path) end end +const _SO_EXTENSIONS = (".so", ".dylib", ".dll") + +function _is_shared_library(path::String) + lp = lowercase(path) + return any(ext -> endswith(lp, ext), _SO_EXTENSIONS) +end + +function _parse_so_header(path::String) + # The .so embeds a JI header written by write_header(ff, 1) at staticdata.c:3530 + # followed by cache_flags and write_mod_list. It lives inside an ELF/MachO/PE + # data section (jl_system_image_data). We locate it by scanning for the JI_MAGIC + # signature with pkgimage=1. + return _parse_so_header_by_scan(path) +end + +function _parse_so_header_by_scan(path::String) + # The .so's own JI header is written by write_header(ff, 1) at staticdata.c:3530 + # with pkgimage=1, followed by cache_flags and write_mod_list. + # It also embeds the .ji data blob (which starts with pkgimage=0 header). + # We need the .so's OWN header (pkgimage=1), not the embedded .ji data. + data = read(path) + + # Scan for JI_MAGIC followed by valid format_version, BOM, and pkgimage=1 + offset = 1 + while true + idx = _find_bytes(data, JI_MAGIC, offset) + idx === nothing && throw(ArgumentError("No JI header (pkgimage=1) found in shared library: $path")) + + # Check: magic(8) + version(2) + bom(2) + ptrsize(1) = 13 bytes minimum after magic + if idx + JI_MAGIC_LEN + 13 > length(data) + offset = idx + 1 + continue + end + + # Try parsing — if it's the right header (pkgimage=1), return it + try + io = IOBuffer(@view data[idx:end]) + hdr = _parse_header_from_io(io, path) + if hdr.pkgimage + # _parse_header_from_io recorded byte offsets via position(io), + # which are RELATIVE to the IOBuffer view (i.e. to `idx`). Shift + # them to ABSOLUTE file offsets so remap() patches the correct + # bytes in the .so instead of clobbering the ELF/MachO headers. + return _shift_header_offsets(hdr, idx - 1) + end + catch + # Not a valid header at this offset — keep scanning + end + + offset = idx + 1 + end +end + +# Return a copy of `hdr` with all recorded file offsets shifted by `base`. +# Used to convert .so-relative offsets (parsed from an IOBuffer view) into +# absolute file offsets. (DepModEntry/WorklistEntry are immutable → rebuild.) +function _shift_header_offsets(hdr::PkgImageHeader, base::Int) + base == 0 && return hdr + wl = WorklistEntry[WorklistEntry(w.name, w.uuid, w.build_id_lo, w._file_offset + base) + for w in hdr.worklist] + rm = DepModEntry[DepModEntry(d.name, d.uuid, d.build_id_hi, d.build_id_lo, + d._file_offset_hi + base, d._file_offset_lo + base) + for d in hdr.required_modules] + return PkgImageHeader( + hdr.format_version, hdr.pointer_size, hdr.build_uname, hdr.build_arch, + hdr.julia_version, hdr.git_branch, hdr.git_commit, hdr.pkgimage, hdr.checksum, + hdr.data_start, hdr.data_end, hdr.cache_flags, wl, rm, + hdr._deplist_start, hdr._deplist_end, hdr._path, + ) +end + +function _find_bytes(haystack::Vector{UInt8}, needle::Vector{UInt8}, start::Int=1) + nlen = length(needle) + hlen = length(haystack) + nlen > hlen && return nothing + for i in start:(hlen - nlen + 1) + match = true + for j in 1:nlen + if haystack[i + j - 1] != needle[j] + match = false + break + end + end + match && return i + end + return nothing +end + function _parse_header_from_io(io::IO, path::String) # ── Section 0: Base header (write_header, staticdata_utils.c:505) ── @@ -105,30 +201,38 @@ function _parse_header_from_io(io::IO, path::String) data_start = read(io, Int64) data_end = read(io, Int64) - # ── Section 1: Cache flags (staticdata.c:3468) ── + # ── Section 1: Cache flags (staticdata.c:3468 / 3531) ── cache_flags = read(io, UInt8) - # ── Section 2: Worklist (write_worklist_for_header, staticdata_utils.c:531) ── - - worklist = _read_module_list(io, false) - - # ── Section 3: Dependency list (write_dependency_list, staticdata_utils.c:563) ── - # This section has variable format. The Julia-side parser reads totbytes - # to know how much to skip. We read totbytes, then skip that many bytes - # minus the 8 bytes we already read for totbytes itself. - - deplist_start = position(io) - totbytes = read(io, UInt64) - # totbytes counts from after itself to the end of the dependency list section - # (includes file records, requires, preferences) - if totbytes > 0 - skip(io, totbytes) + # The .so split-image header (pkgimage=1) has a DIFFERENT, simpler format + # than the .ji incremental header (pkgimage=0): + # + # .ji (pkgimage=0): base_header + cache_flags + worklist + deplist + mod_list + # .so (pkgimage=1): base_header + cache_flags + mod_list + # + # See staticdata.c:3528-3532 for the split-image header vs 3465-3481 for .ji. + + worklist = WorklistEntry[] + deplist_start = Int64(0) + deplist_end = Int64(0) + + if pkgimage_flag == 0 + # ── .ji format: Sections 2-4 ── + + # Section 2: Worklist (write_worklist_for_header, staticdata_utils.c:531) + worklist = _read_module_list(io, false) + + # Section 3: Dependency list (write_dependency_list, staticdata_utils.c:563) + deplist_start = position(io) + totbytes = read(io, UInt64) + if totbytes > 0 + skip(io, totbytes) + end + deplist_end = position(io) end - deplist_end = position(io) - - # ── Section 4: Required modules (write_mod_list, staticdata_utils.c:409) ── + # Section 4 (.ji) / Section 2 (.so): Required modules (write_mod_list) required_modules = _read_module_list(io, true) return PkgImageHeader( diff --git a/src/loader.jl b/src/loader.jl index 224ef3f..514a0c2 100644 --- a/src/loader.jl +++ b/src/loader.jl @@ -116,7 +116,7 @@ function load_package_image(path::String; if register pkg = _infer_pkgid(hdr) - restored = Base.register_restored_modules(sv, pkg, path) + restored = @lock Base.require_lock Base.register_restored_modules(sv, pkg, path) if !run_init # register_restored_modules already ran __init__; we can't undo that. diff --git a/test/fixtures.jl b/test/fixtures.jl new file mode 100644 index 0000000..8e5618f --- /dev/null +++ b/test/fixtures.jl @@ -0,0 +1,86 @@ +# Shared fixture-resolution helpers for the depot/pkgimage-dependent tests. +# +# Several tests need a *real* package-image fixture: a depot directory containing +# compiled/v./**/.{ji,so} +# Locally (dev) we can just scan Base.DEPOT_PATH and find them. On a clean CI +# runner there are no pkgimages, so we let CI provision an extracted bundle +# (exactly the Piccolissimo BuildImage `target/pkgimage/` depot) and point us at +# it via ENV["JSD_FIXTURE_DEPOT"]. +# +# Resolution precedence (see jsd_fixture_depots): +# 1. ENV["JSD_FIXTURE_DEPOT"] (set & non-empty) -> search ONLY that depot +# 2. otherwise -> scan Base.DEPOT_PATH +# +# These helpers are `include`d into each @testitem (TestItemRunner runs every +# testitem in its own module, so the file is included per-item). They return +# `nothing` / an empty vector when no fixture is available; callers then SKIP +# rather than error. The exact skip message used across tests: +const JSD_FIXTURE_SKIP_MSG = + "no .ji/.so fixture found; set JSD_FIXTURE_DEPOT to an extracted bundle depot, " * + "or run in a depot with pkgimages" + +# Ordered list of depot roots to search, honoring JSD_FIXTURE_DEPOT first. +function jsd_fixture_depots() + fixture = get(ENV, "JSD_FIXTURE_DEPOT", "") + if !isempty(fixture) + return [fixture] + end + return collect(Base.DEPOT_PATH) +end + +# The compiled/v. subdir of a depot (may or may not exist). +function jsd_compiled_dir(depot::AbstractString) + return joinpath(depot, "compiled", "v$(VERSION.major).$(VERSION.minor)") +end + +# Find a pkgimage `.so` (across the resolved depots) whose embedded header is a +# pkgimage and has at least one required module. Returns the path or `nothing`. +function jsd_find_so(parse_header) + for depot in jsd_fixture_depots() + compiled = jsd_compiled_dir(depot) + isdir(compiled) || continue + for (root, _, files) in walkdir(compiled) + for f in files + endswith(f, ".so") || continue + cand = joinpath(root, f) + h = try + parse_header(cand) + catch + continue + end + h.pkgimage && !isempty(h.required_modules) && return cand + end + end + end + return nothing +end + +# Collect up to `limit` `.ji` files from the resolved depots. If `predicate` is +# given it is called with the parsed header and must return true to keep the +# file (parse failures are skipped). Returns a (possibly empty) Vector{String}. +function jsd_find_ji(; limit::Integer = 3, parse_header = nothing, predicate = nothing) + ji_files = String[] + for depot in jsd_fixture_depots() + compiled = jsd_compiled_dir(depot) + isdir(compiled) || continue + for (root, _, files) in walkdir(compiled) + for f in files + endswith(f, ".ji") || continue + cand = joinpath(root, f) + if predicate === nothing + push!(ji_files, cand) + else + hdr = try + parse_header(cand) + catch + continue + end + predicate(hdr) && push!(ji_files, cand) + end + length(ji_files) >= limit && return ji_files + end + length(ji_files) >= limit && return ji_files + end + end + return ji_files +end diff --git a/test/test_header.jl b/test/test_header.jl index 525d032..0aeea77 100644 --- a/test/test_header.jl +++ b/test/test_header.jl @@ -3,62 +3,57 @@ using TestItems @testitem "parse real .ji files from depot" begin using JuliaStaticData + include(joinpath(@__DIR__, "fixtures.jl")) - ji_files = String[] - depot = get(ENV, "JULIA_DEPOT_PATH", joinpath(homedir(), ".julia")) - compiled_dir = joinpath(first(split(depot, ':')), "compiled", "v$(VERSION.major).$(VERSION.minor)") - if isdir(compiled_dir) - for (root, dirs, files) in walkdir(compiled_dir) - for f in files - endswith(f, ".ji") && push!(ji_files, joinpath(root, f)) - end - length(ji_files) >= 3 && break - end - end + # Resolve .ji fixtures from JSD_FIXTURE_DEPOT or, failing that, Base.DEPOT_PATH. + ji_files = jsd_find_ji(; limit = 3) - @assert !isempty(ji_files) "No .ji files found in depot at $compiled_dir" + if isempty(ji_files) + @info "SKIP parse real .ji files: $JSD_FIXTURE_SKIP_MSG" + @test_skip JSD_FIXTURE_SKIP_MSG + else + for f in ji_files[1:min(3, length(ji_files))] + hdr = parse_header(f) - for f in ji_files[1:min(3, length(ji_files))] - hdr = parse_header(f) + @test hdr.format_version >= 12 + @test hdr.pointer_size in (4, 8) + @test !isempty(hdr.julia_version) + is_split = isfile(splitext(f)[1] * ".so") || isfile(splitext(f)[1] * ".dylib") + if is_split + # Split images: .ji has pkgimage=false, data offsets are 0 + # (data blob and its offsets live in the companion .so) + @test hdr.data_start == 0 + @test hdr.data_end == 0 + @test !hdr.pkgimage # .ji half always has pkgimage=false - @test hdr.format_version >= 12 - @test hdr.pointer_size in (4, 8) - @test !isempty(hdr.julia_version) - is_split = isfile(splitext(f)[1] * ".so") || isfile(splitext(f)[1] * ".dylib") - if is_split - # Split images: .ji has pkgimage=false, data offsets are 0 - # (data blob and its offsets live in the companion .so) - @test hdr.data_start == 0 - @test hdr.data_end == 0 - @test !hdr.pkgimage # .ji half always has pkgimage=false + # The companion .so should have pkgimage=true and valid data offsets + so_path = splitext(f)[1] * ".so" + if isfile(so_path) + so_hdr = parse_header(so_path) + @test so_hdr.pkgimage + @test so_hdr.data_end > so_hdr.data_start + end + else + # Non-split: data blob is in the .ji itself + @test hdr.data_end > hdr.data_start + end + @test !isempty(hdr.worklist) - # The companion .so should have pkgimage=true and valid data offsets - so_path = splitext(f)[1] * ".so" - if isfile(so_path) - so_hdr = parse_header(so_path) - @test so_hdr.pkgimage - @test so_hdr.data_end > so_hdr.data_start + for w in hdr.worklist + @test !isempty(w.name) + @test w.build_id_lo != 0 end - else - # Non-split: data blob is in the .ji itself - @test hdr.data_end > hdr.data_start - end - @test !isempty(hdr.worklist) - for w in hdr.worklist - @test !isempty(w.name) - @test w.build_id_lo != 0 - end + for d in hdr.required_modules + @test !isempty(d.name) + end - for d in hdr.required_modules - @test !isempty(d.name) + buf = IOBuffer() + inspect(hdr; io=buf) + output = String(take!(buf)) + @test contains(output, "Julia Package Image Header") + @test contains(output, hdr.worklist[end].name) end - - buf = IOBuffer() - inspect(hdr; io=buf) - output = String(take!(buf)) - @test contains(output, "Julia Package Image Header") - @test contains(output, hdr.worklist[end].name) end end diff --git a/test/test_register_lock.jl b/test/test_register_lock.jl new file mode 100644 index 0000000..5c88064 --- /dev/null +++ b/test/test_register_lock.jl @@ -0,0 +1,106 @@ +using TestItemRunner +using TestItems + +# Regression tests for the register-LOCK fix in loader.jl: +# +# restored = @lock Base.require_lock Base.register_restored_modules(sv, pkg, path) +# +# `load_package_image(...; register=true)` hands the deserialized SimpleVector to +# `Base.register_restored_modules`, which mutates Base's GLOBAL module registry +# (`loaded_modules_order`, `loaded_precompiles`, `pkgorigins`). On Julia 1.12 that +# function opens with `assert_havelock(require_lock)`, so: +# +# * Called WITHOUT the lock -> ConcurrencyViolationError (the original bug). +# * Concurrent loads racing on the registry without serialization -> corruption. +# +# The fix takes `Base.require_lock` around the registration call, which both +# satisfies the assertion and serializes concurrent multi-dependency image loads. +# +# These tests exercise `Base.register_restored_modules` directly (the exact call +# the fix wraps) using a SYNTHETIC SimpleVector shaped like what +# `jl_restore_package_image_from_file` / `jl_restore_incremental` return: +# +# sv[1] :: Vector{Any} restored Modules +# sv[2] :: Vector{Any} __init__ callbacks +# sv[3] :: Vector{Any} (extra, unused here) +# +# Using empty inner vectors means registration touches the lock-guarded path +# WITHOUT mutating the live registry with bogus modules — so the tests are +# self-contained and safe to run inside any session. +# +# LIMITATION: A full end-to-end `load_package_image` round-trip is intentionally +# NOT exercised here. That would require a precompiled package-image fixture +# (.ji/.so) that is (a) built with cache flags matching the running session and +# (b) has all of its own dependencies already loaded and resolvable. Arbitrary +# images from the depot fail in the C deserialization step ("Pkgimage flags +# mismatch") *before* the locked registration line is ever reached, so they do +# not test the fix and only make the suite flaky. Pkg.test's sandbox depot has no +# usable images at all (see test_so_remap.jl). We therefore test the exact +# function the fix wraps — `Base.register_restored_modules` — directly, which is +# both deterministic and independent of any depot fixture. + +@testitem "register_restored_modules requires Base.require_lock (the bug the fix prevents)" begin + using JuliaStaticData + + # A minimal SimpleVector with no restored modules and no init callbacks: + # register_restored_modules will run, but mutate nothing observable. + sv = Core.svec(Any[], Any[], Any[]) + pkg = Base.PkgId(Base.UUID("72ac74b6-0a8a-4afc-a67b-32735e8af5c8"), "JSDLockProbe") + path = tempname() * ".ji" # path is only recorded, never read + + # WITHOUT the lock: this is precisely the pre-fix call site. It must throw + # ConcurrencyViolationError via assert_havelock(require_lock) in Base. + @test_throws ConcurrencyViolationError Base.register_restored_modules(sv, pkg, path) + + # WITH the lock (the fix): registration succeeds and returns sv[1]. + restored = @lock Base.require_lock Base.register_restored_modules(sv, pkg, path) + @test restored === sv[1] + @test isempty(restored) +end + +@testitem "concurrent registration through the locked path stays consistent" begin + using JuliaStaticData + + # Simulate many dependency images being loaded+registered concurrently, each + # going through the SAME locked path the loader uses. Without the @lock guard + # every spawned task would hit assert_havelock and raise + # ConcurrencyViolationError; with it they serialize cleanly and the global + # registry is left consistent. + n = 32 + pkgs = [Base.PkgId(Base.UUID(UInt128(i)), "JSDConcProbe$(i)") for i in 1:n] + + # Snapshot the live registry. Our synthetic images register no root modules, + # so these global vectors must be byte-for-byte unchanged afterwards — i.e. + # no double-registration / no spurious or corrupted entries. + order_before = copy(Base.loaded_modules_order) + precompiles_before_len = length(Base.loaded_precompiles) + + errors = Channel{Any}(n) + @sync for i in 1:n + Threads.@spawn begin + sv = Core.svec(Any[], Any[], Any[]) # no root modules, no inits + try + @lock Base.require_lock begin + Base.register_restored_modules(sv, pkgs[i], tempname() * ".ji") + end + catch e + put!(errors, e) + end + end + end + close(errors) + + collected = collect(errors) + # No task may have raised — in particular no ConcurrencyViolationError. + @test isempty(collected) + @test all(e -> !(e isa ConcurrencyViolationError), collected) + + # Registry consistency: nothing registered a root module, so order is intact. + @test Base.loaded_modules_order == order_before + @test length(Base.loaded_precompiles) == precompiles_before_len + + # Sanity: with >1 thread this is a genuine concurrency test; with 1 thread it + # degenerates to a deterministic serial pass through the locked path (still a + # valid regression for the assert_havelock requirement). + @test Threads.nthreads() >= 1 +end diff --git a/test/test_remap.jl b/test/test_remap.jl index 7345fc7..1d5ae7e 100644 --- a/test/test_remap.jl +++ b/test/test_remap.jl @@ -3,152 +3,113 @@ using TestItems @testitem "remap dependency build-id roundtrip" begin using JuliaStaticData - - # Find a .ji file with at least one required module - ji_files = String[] - depot = get(ENV, "JULIA_DEPOT_PATH", joinpath(homedir(), ".julia")) - compiled_dir = joinpath(first(split(depot, ':')), "compiled", "v$(VERSION.major).$(VERSION.minor)") - if isdir(compiled_dir) - for (root, dirs, files) in walkdir(compiled_dir) - for f in files - if endswith(f, ".ji") - try - hdr = parse_header(joinpath(root, f)) - if !isempty(hdr.required_modules) - push!(ji_files, joinpath(root, f)) - end - catch - end - end + include(joinpath(@__DIR__, "fixtures.jl")) + + # Find a .ji file with at least one required module (JSD_FIXTURE_DEPOT or DEPOT_PATH). + ji_files = jsd_find_ji(; limit = 1, parse_header = parse_header, + predicate = hdr -> !isempty(hdr.required_modules)) + + if isempty(ji_files) + @info "SKIP remap dependency build-id roundtrip: $JSD_FIXTURE_SKIP_MSG" + @test_skip JSD_FIXTURE_SKIP_MSG + else + src = ji_files[1] + hdr_before = parse_header(src) + + @test !isempty(hdr_before.required_modules) + dep = hdr_before.required_modules[1] + + # Remap with a known target value + target_id = UInt128(0xDEADBEEFCAFE1234) << 64 | UInt128(0x0123456789ABCDEF) + spec = RemapSpec(dep.name, dep.uuid, target_id) + + tmp = tempname() * ".ji" + try + remap(src, tmp, [spec]) + + hdr_after = parse_header(tmp) + dep_after = hdr_after.required_modules[1] + + @test dep_after.name == dep.name + @test dep_after.uuid == dep.uuid + @test dep_after.build_id_hi == UInt64(0xDEADBEEFCAFE1234) + @test dep_after.build_id_lo == UInt64(0x0123456789ABCDEF) + + # Header metadata unchanged + @test hdr_after.format_version == hdr_before.format_version + @test hdr_after.julia_version == hdr_before.julia_version + @test hdr_after.checksum == hdr_before.checksum # data blob untouched + @test length(hdr_after.worklist) == length(hdr_before.worklist) + @test length(hdr_after.required_modules) == length(hdr_before.required_modules) + + # Non-remapped dependencies unchanged + for i in 2:length(hdr_before.required_modules) + @test hdr_after.required_modules[i].build_id_hi == hdr_before.required_modules[i].build_id_hi + @test hdr_after.required_modules[i].build_id_lo == hdr_before.required_modules[i].build_id_lo end - length(ji_files) >= 1 && break + finally + isfile(tmp) && rm(tmp) end end - - @assert !isempty(ji_files) "No .ji files with required_modules found in depot at $compiled_dir" - - src = ji_files[1] - hdr_before = parse_header(src) - - @test !isempty(hdr_before.required_modules) - dep = hdr_before.required_modules[1] - - # Remap with a known target value - target_id = UInt128(0xDEADBEEFCAFE1234) << 64 | UInt128(0x0123456789ABCDEF) - spec = RemapSpec(dep.name, dep.uuid, target_id) - - tmp = tempname() * ".ji" - try - remap(src, tmp, [spec]) - - hdr_after = parse_header(tmp) - dep_after = hdr_after.required_modules[1] - - @test dep_after.name == dep.name - @test dep_after.uuid == dep.uuid - @test dep_after.build_id_hi == UInt64(0xDEADBEEFCAFE1234) - @test dep_after.build_id_lo == UInt64(0x0123456789ABCDEF) - - # Header metadata unchanged - @test hdr_after.format_version == hdr_before.format_version - @test hdr_after.julia_version == hdr_before.julia_version - @test hdr_after.checksum == hdr_before.checksum # data blob untouched - @test length(hdr_after.worklist) == length(hdr_before.worklist) - @test length(hdr_after.required_modules) == length(hdr_before.required_modules) - - # Non-remapped dependencies unchanged - for i in 2:length(hdr_before.required_modules) - @test hdr_after.required_modules[i].build_id_hi == hdr_before.required_modules[i].build_id_hi - @test hdr_after.required_modules[i].build_id_lo == hdr_before.required_modules[i].build_id_lo - end - finally - isfile(tmp) && rm(tmp) - end end @testitem "remap with no matching spec is no-op" begin using JuliaStaticData - - ji_files = String[] - depot = get(ENV, "JULIA_DEPOT_PATH", joinpath(homedir(), ".julia")) - compiled_dir = joinpath(first(split(depot, ':')), "compiled", "v$(VERSION.major).$(VERSION.minor)") - if isdir(compiled_dir) - for (root, dirs, files) in walkdir(compiled_dir) - for f in files - if endswith(f, ".ji") - try - hdr = parse_header(joinpath(root, f)) - if !isempty(hdr.required_modules) - push!(ji_files, joinpath(root, f)) - end - catch - end - end + include(joinpath(@__DIR__, "fixtures.jl")) + + ji_files = jsd_find_ji(; limit = 1, parse_header = parse_header, + predicate = hdr -> !isempty(hdr.required_modules)) + + if isempty(ji_files) + @info "SKIP remap with no matching spec is no-op: $JSD_FIXTURE_SKIP_MSG" + @test_skip JSD_FIXTURE_SKIP_MSG + else + src = ji_files[1] + hdr_before = parse_header(src) + tmp = tempname() * ".ji" + try + remap(src, tmp, [RemapSpec("NonexistentModule", nothing, UInt128(42))]) + hdr_after = parse_header(tmp) + + for i in eachindex(hdr_before.required_modules) + @test hdr_after.required_modules[i].build_id_hi == hdr_before.required_modules[i].build_id_hi + @test hdr_after.required_modules[i].build_id_lo == hdr_before.required_modules[i].build_id_lo end - length(ji_files) >= 1 && break + finally + isfile(tmp) && rm(tmp) end end - - @assert !isempty(ji_files) "No .ji files with required_modules found in depot" - - src = ji_files[1] - hdr_before = parse_header(src) - tmp = tempname() * ".ji" - try - remap(src, tmp, [RemapSpec("NonexistentModule", nothing, UInt128(42))]) - hdr_after = parse_header(tmp) - - for i in eachindex(hdr_before.required_modules) - @test hdr_after.required_modules[i].build_id_hi == hdr_before.required_modules[i].build_id_hi - @test hdr_after.required_modules[i].build_id_lo == hdr_before.required_modules[i].build_id_lo - end - finally - isfile(tmp) && rm(tmp) - end end @testitem "remap worklist build_id.lo" begin using JuliaStaticData - - ji_files = String[] - depot = get(ENV, "JULIA_DEPOT_PATH", joinpath(homedir(), ".julia")) - compiled_dir = joinpath(first(split(depot, ':')), "compiled", "v$(VERSION.major).$(VERSION.minor)") - if isdir(compiled_dir) - for (root, dirs, files) in walkdir(compiled_dir) - for f in files - if endswith(f, ".ji") - try - hdr = parse_header(joinpath(root, f)) - if !isempty(hdr.worklist) - push!(ji_files, joinpath(root, f)) - end - catch - end - end - end - length(ji_files) >= 1 && break + include(joinpath(@__DIR__, "fixtures.jl")) + + ji_files = jsd_find_ji(; limit = 1, parse_header = parse_header, + predicate = hdr -> !isempty(hdr.worklist)) + + if isempty(ji_files) + @info "SKIP remap worklist build_id.lo: $JSD_FIXTURE_SKIP_MSG" + @test_skip JSD_FIXTURE_SKIP_MSG + else + src = ji_files[1] + hdr_before = parse_header(src) + w = hdr_before.worklist[end] + + target_lo = UInt64(0xAAAABBBBCCCCDDDD) + target_id = UInt128(target_lo) # hi=0, lo=target + spec = RemapSpec(w.name, w.uuid, target_id) + + tmp = tempname() * ".ji" + try + remap(src, tmp, [spec]; remap_worklist=true) + hdr_after = parse_header(tmp) + w_after = hdr_after.worklist[end] + + @test w_after.name == w.name + @test w_after.build_id_lo == target_lo + finally + isfile(tmp) && rm(tmp) end end - - @assert !isempty(ji_files) "No .ji files found in depot" - - src = ji_files[1] - hdr_before = parse_header(src) - w = hdr_before.worklist[end] - - target_lo = UInt64(0xAAAABBBBCCCCDDDD) - target_id = UInt128(target_lo) # hi=0, lo=target - spec = RemapSpec(w.name, w.uuid, target_id) - - tmp = tempname() * ".ji" - try - remap(src, tmp, [spec]; remap_worklist=true) - hdr_after = parse_header(tmp) - w_after = hdr_after.worklist[end] - - @test w_after.name == w.name - @test w_after.build_id_lo == target_lo - finally - isfile(tmp) && rm(tmp) - end end diff --git a/test/test_so_remap.jl b/test/test_so_remap.jl new file mode 100644 index 0000000..e86a8a1 --- /dev/null +++ b/test/test_so_remap.jl @@ -0,0 +1,69 @@ +using TestItemRunner +using TestItems + +# Regression tests for parsing + remapping build-ids in split-image .so files. +# The .so embeds its own JI header (pkgimage=1) inside an ELF/MachO data section. +# parse_header() scans for it and records byte offsets; those offsets MUST be +# absolute file offsets (not relative to the embedded-header location), or remap() +# will patch the wrong bytes — historically clobbering the ELF program headers. + +@testitem "parse_header(.so): recorded offsets are ABSOLUTE and point at the build-id bytes" begin + using JuliaStaticData + include(joinpath(@__DIR__, "fixtures.jl")) + + # Resolve a pkgimage .so with required_modules from the fixture depot + # (JSD_FIXTURE_DEPOT) or, failing that, by scanning Base.DEPOT_PATH. + so = jsd_find_so(parse_header) + if so === nothing + @info "SKIP parse_header(.so) offsets: $JSD_FIXTURE_SKIP_MSG" + @test_skip JSD_FIXTURE_SKIP_MSG + else + hdr = parse_header(so) + + # The decisive invariant: seeking to a recorded offset and reading must + # yield exactly the build-id parse_header reported. (The old idx-relative + # bug made these offsets too small → they pointed into the ELF headers.) + open(so, "r") do io + for d in hdr.required_modules + seek(io, d._file_offset_lo); @test read(io, UInt64) == d.build_id_lo + seek(io, d._file_offset_hi); @test read(io, UInt64) == d.build_id_hi + end + end + # Offsets must lie past the ELF program-header region (sanity vs the bug). + @test minimum(d._file_offset_lo for d in hdr.required_modules) > 64 + end +end + +@testitem "remap(.so): patches build-ids in place, keeps the file structurally intact" begin + using JuliaStaticData + include(joinpath(@__DIR__, "fixtures.jl")) + + so = jsd_find_so(parse_header) + if so === nothing + @info "SKIP remap(.so): $JSD_FIXTURE_SKIP_MSG" + @test_skip JSD_FIXTURE_SKIP_MSG + else + tmp = tempname() * ".so" + cp(so, tmp; force=true) + try + hdr = parse_header(tmp) + dep = hdr.required_modules[1] + target = UInt128(0xABCDEF0123456789) << 64 | UInt128(0x0011223344556677) + remap(tmp, tmp, [RemapSpec(dep.name, dep.uuid, target)]) + + hdr2 = parse_header(tmp) # must still parse (ELF intact) + d2 = hdr2.required_modules[1] + @test d2.name == dep.name + @test d2.build_id_hi == UInt64(0xABCDEF0123456789) + @test d2.build_id_lo == UInt64(0x0011223344556677) + open(tmp, "r") do io + seek(io, d2._file_offset_lo); @test read(io, UInt64) == UInt64(0x0011223344556677) + end + for i in 2:length(hdr.required_modules) + @test hdr2.required_modules[i].build_id_lo == hdr.required_modules[i].build_id_lo + end + finally + isfile(tmp) && rm(tmp) + end + end +end