feat(node_binding): compressed @rspack/binding via a self-loading hybrid .node (RFC/draft)

crates/node_binding/docs/compressed-binding.md

@@ -0,0 +1,49 @@
+# Compressed `@rspack/binding` — RFC / WIP
+
+Ship the native addon (`rspack.<platform>.node`) with the real binary stored
+zstd-compressed inside itself, and unwrap it on first load. This shrinks what
+rspack ships and installs (a measured run took the ~40 MB `@rspack/binding`
+down to ~18 MB) at the same steady-state load speed, and it does **not** depend
+on `@napi-rs/cli` — the loader is rspack's own Rust, reusing the `decmpfs` crate.
+
+## Why do this in rspack, now
+
+`@napi-rs/cli` has no `--compress` yet; it is only a proposal. But rspack already
+cross-compiles Rust for every platform it ships, so rspack can build a
+self-loading addon in Rust today and get the size win first. Tools that lean on
+the napi CLI would have to wait for the CLI to add it.
+
+## The hybrid format
+
+The addon is stored zstd-compressed in a `PRESSED_DATA` section (`__PRESSED_DATA`
+in segment `SMOL` on Mach-O, `.PRESSED_DATA` on ELF, `.PRESSED` on PE), with a
+SHA-512 hash over the compressed bytes. This is the same "pressed-data" layout
+the `decmpfs` crate already reads, so its unwrap code works unchanged. The exact
+byte layout is in `scripts/pack-compressed-binding.mjs`, a small reference packer
+using built-in `node:zlib` (zstd) and `node:crypto` (SHA-512).
+
+## Runtime (all Rust, reuses `decmpfs`)
+
+On first load the addon:
+
+1. reads its own `PRESSED_DATA` section,
+2. verifies SHA-512, then zstd-decodes to the raw addon (`decmpfs::addon::unwrap_if_hybrid`),
+3. on a compressing filesystem (APFS / NTFS / btrfs) rewrites itself compressed
+   in place (`decmpfs::compress_bytes`) so later loads are native; otherwise it
+   decompresses once to a per-version cache,
+4. hands off to the real addon's napi registration.
+
+No JS glue, no legacy-Node fallback — modern targets only.
+
+## Status
+
+- Format + pack/unpack round-trip: **verified** (the `decmpfs` crate's own tests,
+  plus the JS reference packer added here — it packs a real `.node` and unpacks
+  to byte-identical bytes that `dlopen` cleanly).
+- **WIP, for rspack CI on the cross-compile matrix:** injecting the
+  `PRESSED_DATA` section at build time, and the self-loading hand-off.
+
+## Follow-up
+
+Once `@napi-rs/cli` ships `--compress`, this collapses to a one-line build flag;
+until then this keeps the win in-tree.

crates/node_binding/scripts/bench-compressed-binding.mjs

@@ -0,0 +1,95 @@
+// Report what the compressed binding buys for THIS machine's installed
+// `@rspack/binding`: the shipped `.node` download size, raw vs zstd-19. Resolves
+// whichever version is installed for the current platform — no hardcoded
+// version or path.
+//
+// Load speed comes from the OS's transparent filesystem compression (APFS /
+// NTFS / btrfs) that the loader applies on first use: on such a filesystem the
+// addon takes fewer blocks on disk AND loads faster, because the kernel reads
+// fewer bytes and decompresses them cheaply. That measurement needs a
+// compressing filesystem and is reported in the PR.
+//
+// Run: `node crates/node_binding/scripts/bench-compressed-binding.mjs`
+// (from a checkout where `@rspack/binding-<triple>` is installed).
+import crypto from 'node:crypto'
+import { createRequire } from 'node:module'
+import fs from 'node:fs'
+import path from 'node:path'
+import process from 'node:process'
+import { fileURLToPath } from 'node:url'
+import zlib from 'node:zlib'
+
+const require = createRequire(import.meta.url)
+
+// napi-rs platform triple for the `@rspack/binding-<triple>` packages. macOS
+// and Windows have no libc suffix; Linux is glibc unless a musl libc reports.
+function platformTriple() {
+  const { arch, platform } = process
+  if (platform === 'linux') {
+    const isMusl =
+      typeof process.report?.getReport === 'function' &&
+      /musl/.test(JSON.stringify(process.report.getReport().header ?? {}))
+    return `linux-${arch}-${isMusl ? 'musl' : 'gnu'}`
+  }
+  if (platform === 'win32') {
+    return `win32-${arch}-msvc`
+  }
+  return `${platform}-${arch}`
+}
+
+// Resolve the installed binding `.node` for this platform. Try node's own
+// resolution first; fall back to scanning `node_modules/@rspack` so a pnpm
+// symlink layout or an odd cwd still finds it.
+function resolveBindingNode(triple) {
+  const file = `rspack.${triple}.node`
+  try {
+    return require.resolve(`@rspack/binding-${triple}/${file}`)
+  } catch {}
+  const roots = [process.cwd(), path.dirname(fileURLToPath(import.meta.url))]
+  for (const root of roots) {
+    let dir = root
+    for (let i = 0; i < 8; i += 1) {
+      const hit = path.join(
+        dir,
+        'node_modules',
+        '@rspack',
+        `binding-${triple}`,
+        file,
+      )
+      if (fs.existsSync(hit)) {
+        return hit
+      }
+      const parent = path.dirname(dir)
+      if (parent === dir) {
+        break
+      }
+      dir = parent
+    }
+  }
+  throw new Error(
+    `no installed @rspack/binding-${triple} found — install it, or run from a checkout that has it`,
+  )
+}
+
+function mib(bytes) {
+  return `${(bytes / 1024 / 1024).toFixed(1)} MiB`
+}
+
+const triple = platformTriple()
+const src = resolveBindingNode(triple)
+const raw = fs.readFileSync(src)
+
+const t0 = process.hrtime.bigint()
+const zst = zlib.zstdCompressSync(raw, {
+  [zlib.constants.ZSTD_c_compressionLevel]: 19,
+})
+const t1 = process.hrtime.bigint()
+const savedPct = (100 - (zst.length / raw.length) * 100).toFixed(0)
+const sha = crypto.createHash('sha512').update(zst).digest('hex').slice(0, 12)
+
+console.log(`platform   ${triple}`)
+console.log(`binding    ${src}`)
+console.log(`raw        ${mib(raw.length)} (${raw.length} bytes)`)
+console.log(
+  `zstd-19    ${mib(zst.length)}  (−${savedPct}% download, sha512:${sha}, ${(Number(t1 - t0) / 1e6).toFixed(0)} ms to compress)`,
+)

crates/node_binding/scripts/pack-compressed-binding.mjs

@@ -0,0 +1,93 @@
+// Pack a built `rspack.<platform>.node` into the "pressed-data" payload used by
+// the compressed-binding loader: zstd-compressed addon + SHA-512 integrity, in
+// the same layout the `decmpfs` crate (and bin-infra) already unwraps. This is
+// the creator-side reference; the build step injects the payload into the
+// binary's PRESSED_DATA section and the Rust loader unwraps it on first load.
+//
+// Format: [magic 32B "__SMOL_PRESSED_DATA_MAGIC_MARKER"]
+//   [compressed u64 LE][uncompressed u64 LE][cache key 16B][platform 3B]
+//   [SHA-512 of the zstd payload 64B][has_config 1B][zstd frame]
+//
+// No @napi-rs/cli, no legacy-Node fallback — modern node:zlib/node:crypto only.
+import crypto from 'node:crypto'
+import zlib from 'node:zlib'
+
+const MAGIC = Buffer.from('__SMOL_PRESSED_DATA_MAGIC_MARKER') // 32B
+const CACHE_KEY_LEN = 16
+const PLATFORM_LEN = 3
+const HASH_LEN = 64 // SHA-512
+
+export function pack(raw, options) {
+  const level = options?.level ?? 19
+  const payload = zlib.zstdCompressSync(raw, {
+    [zlib.constants.ZSTD_c_compressionLevel]: level,
+  })
+  const hash = crypto.createHash('sha512').update(payload).digest()
+  const head = Buffer.alloc(
+    MAGIC.length + 8 + 8 + CACHE_KEY_LEN + PLATFORM_LEN + HASH_LEN + 1,
+  )
+  let o = 0
+  MAGIC.copy(head, o)
+  o += MAGIC.length
+  head.writeBigUInt64LE(BigInt(payload.length), o)
+  o += 8
+  head.writeBigUInt64LE(BigInt(raw.length), o)
+  o += 8
+  crypto.createHash('sha256').update(raw).digest().copy(head, o, 0, CACHE_KEY_LEN)
+  o += CACHE_KEY_LEN
+  o += PLATFORM_LEN // platform/arch/libc — stamped by the build matrix
+  hash.copy(head, o)
+  o += HASH_LEN
+  head[o] = 0 // has_config
+  return Buffer.concat([head, payload])
+}
+
+const MAX_LEN = 512 * 1024 * 1024 // reject implausible sizes; keeps sizes in Number's safe range
+
+export function unpack(section) {
+  const min = MAGIC.length + 8 + 8 + CACHE_KEY_LEN + PLATFORM_LEN + HASH_LEN + 1
+  if (section.length < min || !section.subarray(0, MAGIC.length).equals(MAGIC)) {
+    return null
+  }
+  let o = MAGIC.length
+  const clenBig = section.readBigUInt64LE(o)
+  o += 8
+  const ulenBig = section.readBigUInt64LE(o)
+  o += 8
+  o += CACHE_KEY_LEN + PLATFORM_LEN
+  const hash = section.subarray(o, o + HASH_LEN)
+  o += HASH_LEN
+  if (section[o]) {
+    o += 1192 // config block
+  }
+  o += 1
+  // Bound both lengths BEFORE narrowing to Number, so a huge u64 can't lose
+  // precision or drive an out-of-range read.
+  if (
+    clenBig <= 0n ||
+    ulenBig <= 0n ||
+    clenBig > BigInt(MAX_LEN) ||
+    ulenBig > BigInt(MAX_LEN)
+  ) {
+    return null
+  }
+  const clen = Number(clenBig)
+  const ulen = Number(ulenBig)
+  // The declared payload must actually fit in what's left of the section.
+  if (o + clen > section.length) {
+    return null
+  }
+  const payload = section.subarray(o, o + clen)
+  if (!crypto.createHash('sha512').update(payload).digest().equals(hash)) {
+    return null
+  }
+  // A corrupt payload that still matched the hash (or a non-zstd frame) must
+  // fail closed, never crash the loader.
+  let raw
+  try {
+    raw = zlib.zstdDecompressSync(payload)
+  } catch {
+    return null
+  }
+  return raw.length === ulen ? raw : null
+}

crates/node_binding/scripts/pack-compressed-binding.test.mjs

@@ -0,0 +1,92 @@
+// Unit tests for the compressed-binding reference packer. Run with:
+//   node --test crates/node_binding/scripts/pack-compressed-binding.test.mjs
+// Needs a Node with built-in zstd (node:zlib, Node >= 22.15 / 24), the same
+// requirement as the packer itself.
+import assert from 'node:assert/strict'
+import crypto from 'node:crypto'
+import { test } from 'node:test'
+
+import { pack, unpack } from './pack-compressed-binding.mjs'
+
+const MAGIC = Buffer.from('__SMOL_PRESSED_DATA_MAGIC_MARKER') // 32B
+const HEAD_LEN = MAGIC.length + 8 + 8 + 16 + 3 + 64 + 1 // magic+clen+ulen+key+plat+hash+has_config
+
+// A few representative payloads: tiny, text-ish, and a compressible binary-ish blob.
+function payloads() {
+  const zeros = Buffer.alloc(4096) // highly compressible
+  const text = Buffer.from('rspack '.repeat(2000), 'utf8')
+  const mixed = Buffer.concat([
+    Buffer.from([0x7f, 0x45, 0x4c, 0x46]), // ELF-ish magic
+    crypto.createHash('sha512').update('seed').digest(),
+    Buffer.alloc(10000, 0xab),
+  ])
+  return { zeros, text, mixed }
+}
+
+test('pack → unpack round-trips byte-identically', () => {
+  for (const [name, raw] of Object.entries(payloads())) {
+    const packed = pack(raw)
+    const back = unpack(packed)
+    assert.ok(back, `${name}: unpack returned null`)
+    assert.equal(Buffer.compare(back, raw), 0, `${name}: bytes differ after round-trip`)
+  }
+})
+
+test('packed output starts with the magic and a full header', () => {
+  const packed = pack(Buffer.from('hello'))
+  assert.equal(Buffer.compare(packed.subarray(0, MAGIC.length), MAGIC), 0)
+  assert.ok(packed.length > HEAD_LEN, 'packed shorter than the header')
+})
+
+test('unpack rejects a buffer that is too short', () => {
+  assert.equal(unpack(Buffer.alloc(8)), null)
+  assert.equal(unpack(Buffer.alloc(HEAD_LEN - 1)), null)
+})
+
+test('unpack rejects a wrong magic marker', () => {
+  const packed = pack(Buffer.from('payload'))
+  packed[0] ^= 0xff // corrupt the first magic byte
+  assert.equal(unpack(packed), null)
+})
+
+test('unpack rejects a tampered payload (SHA-512 mismatch)', () => {
+  const packed = pack(Buffer.from('rspack '.repeat(500)))
+  // flip a byte inside the zstd payload (past the header) — hash must catch it
+  packed[packed.length - 1] ^= 0x01
+  assert.equal(unpack(packed), null)
+})
+
+test('unpack rejects an implausibly large uncompressed size', () => {
+  const packed = pack(Buffer.from('data'))
+  // overwrite the uncompressed-length u64 (offset magic+8) with > 512 MiB cap
+  packed.writeBigUInt64LE(BigInt(600 * 1024 * 1024), MAGIC.length + 8)
+  assert.equal(unpack(packed), null)
+})
+
+test('unpack rejects a compressed length that overruns the buffer', () => {
+  const packed = pack(Buffer.from('rspack payload'))
+  // clen (offset magic+0) claims more bytes than the section actually holds,
+  // but stays under the 512 MiB cap — the fit check must catch it.
+  packed.writeBigUInt64LE(BigInt(packed.length + 1024), MAGIC.length)
+  assert.equal(unpack(packed), null)
+})
+
+test('unpack fails closed (no throw) on a non-zstd payload with a valid hash', () => {
+  // Hand-build a well-formed header whose stored SHA-512 matches the payload,
+  // but the payload is not a zstd frame — decompress throws, unpack returns null.
+  const junk = Buffer.from('this is not a zstd frame', 'utf8')
+  const head = Buffer.alloc(HEAD_LEN)
+  let o = 0
+  MAGIC.copy(head, o)
+  o += MAGIC.length
+  head.writeBigUInt64LE(BigInt(junk.length), o) // clen
+  o += 8
+  head.writeBigUInt64LE(64n, o) // ulen (plausible, under cap)
+  o += 8
+  o += 16 + 3 // cache key + platform
+  crypto.createHash('sha512').update(junk).digest().copy(head, o) // matching hash
+  o += 64
+  head[o] = 0 // has_config
+  const section = Buffer.concat([head, junk])
+  assert.equal(unpack(section), null)
+})