diff --git a/.changeset/tar-xz-streaming.md b/.changeset/tar-xz-streaming.md new file mode 100644 index 0000000..d2553db --- /dev/null +++ b/.changeset/tar-xz-streaming.md @@ -0,0 +1,42 @@ +--- +'tar-xz': minor +--- + +True streaming for Node `extract()` and `list()` — memory now stays O(largest single entry) instead of O(archive). Delivers the "planned optimization" advertised by v6.0.0's stream-first README. + +```ts +// 1 GB .tar.xz with 100 MB files → peak heap ~100 MB, not ~1 GB +for await (const entry of extract(bigArchive)) { + for await (const chunk of entry.data) { + consumer(chunk); // chunks arrive as XZ decompresses, no pre-buffer + } +} +``` + +## What changed under the hood + +- New shared `parseTar(source, mode)` AsyncGenerator coroutine in `tar-parser.ts`. The old `TarUnpack`/`TarList` `Writable` classes are gone; they were artificially-buffered incremental parsers that never received true streaming input. +- `extract()` and `list()` rewired to consume `parseTar` directly via `for await`, with the consumer's iteration controlling decompression rate (natural backpressure — no explicit pause/resume). +- Deprecated helpers `collectAllChunks` / `decompressXz` / `runWritable` removed from `xz-helpers.ts` (replaced by `streamXz()`). +- New PAX-bomb DoS guard: `MAX_PAX_HEADER_BYTES = 1 MB`. Crafted archives declaring giant PAX headers throw `Error` with `code === 'TAR_PARSER_INVARIANT'`. + +## Behavior changes (non-breaking but observable — Hyrum's Law) + +These were always part of the streaming contract advertised by v6.0.0 but only become observable now: + +1. **`entry.data` chunk count and size** now reflect the XZ decompressor's output cadence (typically 64 KB blocks). Previously: exactly one chunk per entry containing the full file. Code that assumed "one chunk per entry" or relied on `Buffer.concat` of all chunks should iterate normally — `entry.bytes()` and `entry.text()` still collect the whole file. +2. **`entry.bytes()` and `entry.text()` memoize.** Calling `bytes()` twice now returns the same buffer reference (efficient) instead of re-collecting. Memory promise preserved: at most one entry's bytes are held at a time. +3. **`entry.data` is single-use.** Iterating it twice now yields nothing on the second pass (was: same single chunk both times). If you need to re-read, call `bytes()` first and keep the reference. +4. **Consumer-break stops iteration cleanly.** If you `for await ... break`, the generator closes and no further errors are surfaced — whether or not the archive is corrupt past the break point. This matches the JS AsyncGenerator convention (`tar-stream`, `it-tar`) where errors beyond the consumer's read horizon are silently discarded. Errors during active forward iteration still throw normally. + +If any of these changes break your code, please open an issue — we're prepared to ship 7.0.0 with a deprecation cycle if a reasonable reliance is demonstrated. + +## Security + +- POSIX path remains fd-based with `O_NOFOLLOW` — TOCTOU window unchanged. +- Windows fallback: by-path operations, so the symlink-swap window now scales with entry size in streaming mode. Mitigation: extract to a directory not writable by other processes. Win32 fd-based extraction tracked as a separate follow-up. +- See `Security model` section in the README for details. + +## Native parity (out of scope this release) + +The streaming refactor only affects the Node path. The browser/WASM path uses a different decompression model and remains buffered (a separate optimization). diff --git a/TODO.md b/TODO.md index 86c1319..94acc97 100644 --- a/TODO.md +++ b/TODO.md @@ -2,7 +2,16 @@ ## In Progress -_None_ +- [ ] 🟡 [tar-xz] True streaming for Node `extract()`/`list()` — Priority: M (started 2026-04-28, branch feat/tar-xz-streaming, story TAR-XZ-STREAMING-2026-04-28) + - [x] ✅ Block 1: `streamXz()` added to `xz-helpers.ts`, old helpers `@deprecated`-tagged, 9 tests in `test/xz-helpers.spec.ts` — all quality gates pass (2026-04-28) + - [x] ✅ Block 2: `parseTar()` AsyncGenerator + `ParseEvent` type added to `tar-parser.ts`; v8-ignore removed from lines 86-89/148-153/169-174 (now hot paths); 8 tests in `test/tar-parser-stream.spec.ts` covering 128-byte chunking, PAX split (S-05), PAX_GLOBAL split (L-M-03), EOA detection, truncation (S-09), list/no-chunk, auto-drain, TAR_PARSER_INVARIANT (D-5); MAX_PAX_HEADER_BYTES=1MB DoS guard (A-07); all gates pass (2026-04-28) + - [x] ✅ Block 3: `extract.ts` rewritten — `TarUnpack` class replaced by lean async generator using `parseTar`; `makeTarEntryWithData` accepts pull-callback with bytes() memoization (D-3); lookahead buffer pattern for correct event routing; S-08 auto-drain + S-08b consumer-break + memoization tests added to `coverage.spec.ts`; all gates pass (2026-04-28) + - [x] ✅ Block 4: `list.ts` rewritten — `TarList` class replaced by 4-line generator using `parseTar(xzStream, 'list')`; S-12 placeholder test added; all gates pass (2026-04-28) + - [x] ✅ Cleanup: `collectAllChunks`, `decompressXz`, `runWritable` removed from `xz-helpers.ts` (zero callers verified); T-06 deprecated-helpers test removed from `xz-helpers.spec.ts`; all gates pass (2026-04-28) + - [x] ✅ Block 5: security regression gate (`test/security.spec.ts` — 18 TOCTOU vectors + S-14 + S-15) + memory-shape CI gate (`test/memory-shape.spec.ts` — 3 high-water tests); vitest.config.ts pool=forks+--expose-gc; file.ts @security TSDoc; README security model subsection (2026-04-28) + - [x] ✅ PR #113 Fix Round 1: 11 findings fixed (F-1 streamXz/parseTar early-termination cleanup; F-2 bytes()-after-iter throw guard; F-3 text() reverted to Buffer.toString for base64/hex/latin1 support; F-4 concurrent dataGen guard; F-5 stray-chunk TAR_PARSER_INVARIANT throw; F-6 memory-shape Test1 preset:6; F-7 vitest.memory.config.ts + test:memory script isolated; F-8 math comment fix; F-9 README wording; F-10 changeset wording; F-11 duplicate §12 header removed); 12 new tests; all 6 quality gates green (2026-04-28) + - [x] ✅ PR #113 Fix Round 2: CR2-1 (M) streamXz() lazy pipeline — moved pipeline()/createUnxz() inside async generator body so no I/O starts before first .next(); T-07 lazy test (readCount=0 + no unhandled rejection); CR2-2 (L) bytes() alloc-once via entry.size — single Uint8Array pre-alloc + in-place set() per chunk, throws TAR_PARSER_INVARIANT on overrun, special-cases entry.size===0; concatChunks() dead-code deleted; all 6 quality gates green: build=0 tsc=0 lint=0 test=0(151) memory=0(3) full=0(489) (2026-04-29) + - [x] ✅ PR #113 CI hotfix: dataWrapper retyped from `AsyncGenerator` to `AsyncIterable` in extract.ts:makeTarEntryWithData — avoids the `[Symbol.asyncDispose]` requirement TS 6 added to AsyncGenerator (lib.esnext Explicit Resource Management). Wrapper restructured: pure AsyncIterable whose `[Symbol.asyncIterator]()` sets dataIterStarted flag and returns dataGen directly (smaller surface, exactly matches public TarEntryWithData.data type). All 6 gates green. (2026-04-29) ## Pending - HIGH @@ -10,7 +19,8 @@ _None_ ## Pending - MEDIUM -- [ ] [tar-xz] True streaming for Node `extract()`/`list()` — replace `Buffer.concat` accumulation (extract.ts:59,91 + list.ts:26) with incremental header→content parsing so memory stays O(largest entry) instead of O(archive). Public README v6.0.0 advertises this as a "planned optimization" — Priority: M +- [ ] [tar-xz] True streaming for Node `extract()`/`list()` — replace `Buffer.concat` accumulation (extract.ts:59,91 + list.ts:26) with incremental header→content parsing so memory stays O(largest entry) instead of O(archive). Public README v6.0.0 advertises this as a "planned optimization" — Priority: M (now in progress as TAR-XZ-STREAMING-2026-04-28 / branch feat/tar-xz-streaming) +- [ ] [Win32] Handle-based extraction (CreateFileW + FILE_FLAG_OPEN_REPARSE_POINT) for `tar-xz` Node `extractFile` — close the TOCTOU window on Windows surfaced by streaming refactor in PR #113. POSIX uses fd-based O_NOFOLLOW; Windows currently falls back to by-path `pipeline(Readable.from(entry.data), createWriteStream(target))` which exposes a wallclock-long symlink-swap window. Estimate ~1-2h. Match node-tar's gradual approach. Priority: M (post-#113). ## Pending - LOW (Nice to Have) diff --git a/docs/plans/TAR-XZ-STREAMING-2026-04-28.md b/docs/plans/TAR-XZ-STREAMING-2026-04-28.md new file mode 100644 index 0000000..6621ec7 --- /dev/null +++ b/docs/plans/TAR-XZ-STREAMING-2026-04-28.md @@ -0,0 +1,894 @@ +--- +doc-meta: + status: draft + story_id: TAR-XZ-STREAMING-2026-04-28 + created: 2026-04-28 + adversarial_applied: true + adversarial_date: 2026-04-28 + llm_consensus_applied: true + llm_consensus_date: 2026-04-28 + llm_consensus_llms: [codex, copilot] + llm_consensus_gemini_status: errored +--- + +# TAR-XZ True Streaming Refactor + +## §1 Scope + +**What changes:** Replace the three-stage O(archive) accumulation pipeline in +`packages/tar-xz/src/node/xz-helpers.ts` and the `Writable`-based `TarUnpack`/`TarList` +classes with a unified `async function* parseTar()` generator that processes the XZ +decompression stream incrementally, yielding entries as they are parsed without holding the +full archive in memory. + +**What stays the same:** +- Public API: `extract(input, options?)` and `list(input)` signatures unchanged + (`AsyncIterable`) +- `TarEntryWithData` interface: `data`, `bytes()`, `text()` unchanged +- `tar-xz/file` subpath helpers (`extractFile`, `listFile`, `createFile`) — NOT modified +- `parseNextHeader()` and all `src/tar/` utilities — reused as-is, NOT modified +- Browser/WASM path (`src/browser/`, `src/index.browser.ts`) — out of scope +- `create()` function — out of scope (already streaming) + +**Memory target:** O(largest single entry). For a 1 GB archive with 100 MB entries, +peak heap ≈ 100 MB, not ≈ 1 GB. + +--- + +## §2 Reality Constraints + +### §2.1 Security surface — 18 TOCTOU vectors must remain closed + +PR #108 (2026-04-27) closed 18 security vectors in `packages/tar-xz/src/node/file.ts`. +All are enforced in `extractFile()`, which operates on `TarEntryWithData` objects yielded +by `extract()`. The refactor touches `extract.ts`, `list.ts`, and `xz-helpers.ts` only. +`file.ts` is NOT modified. + +Security invariants that depend on correct parser output: +- **V1/R6-1** — leaf symlink check: requires correct `entry.type` +- **R4-2/R5-1** — hardlink TOCTOU: requires correct `entry.linkname` +- **V6a/V6b** — NUL/empty rejection: requires `entry.name` faithful to PAX `path` attribute +- **F-001** — traversal check: requires full PAX-extended names (not 100-char truncated names) + +All validated against parsed `TarEntry` fields. Parser is unchanged. + +### §2.2 PAX/GNU format correctness + +`tar-parser.ts:consumePaxHeader()` (line 139–159) already implements re-entrancy: when PAX +payload is split across XZ chunk boundaries, it returns the header block to `state.buffer` +and emits `need-more-data`. This code is correct but currently unreachable (single-buffer +feed). Streaming will promote it to a hot path. Must be covered by S-05 test. + +### §2.3 Backpressure semantics + +Consumer `for await` on `extract()` controls decompression rate. Consumer must drain +`entry.data` before advancing to next entry (same contract as tar-stream/it-tar). If +consumer does not drain, the `parseTar` generator must drain internally before parsing the +next header (S-08). No explicit pause/resume needed — `AsyncGenerator` suspension handles +backpressure naturally. + +### §2.4 Out of scope + +- Browser/WASM path +- Sparse file support (GNU sparse headers) +- GNU `@LongLink` extension (PAX is the only extension in current code) +- Concurrent entry consumption (explicitly forbidden) + +--- + +## §3 Current State + +### §3.1 The accumulation chain — three stages, all O(archive) + +**Stage 1** — `xz-helpers.ts:12–27` `collectAllChunks()`: +Collects ALL compressed bytes into one `Uint8Array` before decompression begins. + +**Stage 2** — `xz-helpers.ts:30–64` `decompressXz()`: +Decompresses ALL bytes into a second `Uint8Array`. For 1 GB `.tar.xz` (10:1 ratio): +produces ≈10 GB on heap. + +**Stage 3** — `xz-helpers.ts:67–74` `runWritable()`: +Writes the entire decompressed buffer in one `writable.write()` call. The +`Buffer.concat([this.state.buffer, chunk])` at `extract.ts:59` fires with an empty left +side — the real problem is stages 1 and 2. + +**Stage 4** (within-entry) — `extract.ts:91` `Buffer.concat(this.contentChunks)`: +Concats chunks for the current entry. Eliminated in streaming model (chunks yielded +directly). `list.ts` has no equivalent (content is discarded). + +### §3.2 Parser state machine — already streaming-ready + +`tar-parser.ts:parseNextHeader()` (line 83–133): +- Returns `need-more-data` when `state.buffer.length < BLOCK_SIZE` (line 87) +- Returns header block to buffer on PAX split (line 151–153) +- Advances `state.buffer` via zero-copy `subarray()` +- All state in `HeaderParserState` — reentrant and stateless across calls + +`extract.ts:processBuffer()` (line 121–130) and `list.ts:processBuffer()` (line 55–63) +are correctly structured incremental parsers receiving artificially-complete input. + +**Conclusion:** The parser does not change. Only the data feed mechanism changes. + +### §3.3 `TarEntryWithData.data` — type contract already correct + +`extract.ts:makeTarEntryWithData()` (line 19–40) returns: +```typescript +data: (async function* () { if (u8.length > 0) yield u8; })() +``` +`file.ts:extractFile` iterates it with `for await (const chunk of entry.data)` (line 297). +The streaming refactor changes what the generator yields, not the type. Zero API changes. + +### §3.4 `v8 ignore` blocks that streaming will activate + +`tar-parser.ts:86–89` — chunk boundary splitting a 512-byte header (currently unreachable) +`tar-parser.ts:148–153` — PAX payload split across XZ chunks (currently unreachable) + +After streaming refactor, both become hot paths. The `v8 ignore` annotations should be +removed in Block 2. + +--- + +## §4 Design Strategy + +### §4.1 Streaming TAR parser — AsyncGenerator coroutine + +**ParseEvent discriminated union:** +```typescript +type ParseEvent = + | { kind: 'entry'; entry: TarEntry } + | { kind: 'chunk'; data: Uint8Array } + | { kind: 'end' } +``` + +**`parseTar(source, mode)` state machine phases:** + +``` +HEADER: + Pull chunks from XZ source until state.buffer.length >= 512 + Call parseNextHeader(state) + → need-more-data : fetch next XZ chunk, concat to state.buffer, retry + → end-of-archive : emit { kind: 'end' }, return + → pax-consumed : loop + → entry : emit { kind: 'entry', entry } + transition to CONTENT (mode='extract', size>0) + or SKIP (mode='list', or size===0) + +CONTENT (extract, size > 0): + Yield { kind: 'chunk', data } for bytes of this entry as they arrive + Split chunk at entry boundary: yield entry portion, keep remainder in state.buffer + When bytesRemaining === 0: transition to PADDING + +SKIP (list, or size === 0): + Consume bytesToSkip bytes silently (no yield) + Split-chunk logic mirrors CONTENT + When bytesToSkip === 0: transition to PADDING (if size > 0) or HEADER + +PADDING: + Consume paddingRemaining bytes silently + When paddingRemaining === 0: transition to HEADER +``` + +**`extract()` assembly:** +1. Pull `kind: 'entry'` → get header, apply strip/filter +2. Yield `TarEntryWithData` with `entry.data` = inner generator pulling `kind: 'chunk'` +3. Outer generator is suspended while consumer iterates `entry.data` +4. When consumer advances, outer generator internally drains remaining chunks (S-08 safety) +5. Pull next `kind: 'entry'` or `kind: 'end'` + +### §4.2 Buffer management + +Bounded carryover in `state.buffer`: +- HEADER phase: max 511 bytes (waiting for one 512-byte block) +- PAX phase: max `paxSize + paxPadding` (typically < 4 KB) +- CONTENT/SKIP/PADDING: max `chunk.length` of one XZ output chunk + +The `Buffer.concat([headerBlock, state.buffer])` re-entrancy in `consumePaxHeader()` (line +151) is the only retained concat. It is bounded by PAX payload size, not archive size. + +### §4.3 Backpressure + +Pipeline: `[XZ input] → createUnxz() Transform → AsyncGenerator.next() pulls` + +`parseTar` uses `for await (const chunk of xzAsyncIterable)`. The generator only +`await`s the next XZ chunk when it needs more bytes. This only happens when the consumer +calls `.next()` on the outer `extract()` generator. Natural demand-driven flow — no +explicit pause/resume or highWaterMark manipulation. + +### §4.4 `list()` vs `extract()` — shared core, distinct strategies + +Both use `parseTar()`. In `list` mode, the generator transitions directly to SKIP after +each entry header without yielding `kind: 'chunk'` events. `list()` collects only +`kind: 'entry'` events and yields them as `TarEntry`. Memory bounded at O(BLOCK_SIZE). + +XZ multi-block skip optimization (jumping past content at XZ block boundaries) is deferred +to §8 (requires encoder-side multi-block support). + +--- + +## §5 BDD Scenarios + +### S-01 — Small archive, single file +``` +Given: .tar.xz with one 1 KB file "hello.txt" +When: extract(input) is iterated +Then: one TarEntryWithData yielded, entry.name === 'hello.txt' +And: await entry.bytes() returns correct 1 KB content +And: peak heapUsed delta < 10 MB +``` + +### S-02 — Large single entry (memory shape proof) +``` +Given: .tar.xz with one 200 MB file "big.bin" +When: extract(input) is iterated, entry.data drained chunk-by-chunk +Then: peak heapUsed delta < 50 MB +And: total bytes from entry.data === 200 * 1024 * 1024 +``` + +### S-03 — Multi-entry archive, ordered +``` +Given: archive [a.txt (10 KB), b.txt (5 MB), c.txt (3 KB)] +When: extract(input) iterated +Then: entries arrive in creation order, content correct +And: peak heap never exceeds 5 MB + buffer overhead +``` + +### S-04 — PAX extended name (> 100 chars) +``` +Given: archive with file whose name is 260 chars +When: extract(input) iterated +Then: entry.name === full 260-char name (PAX path applied) +And: content correct +``` + +### S-05 — PAX header split across XZ chunks +``` +Given: XZ stream mocked to emit 513-byte chunks +And: PAX header payload starts at byte 512 (split) +When: extract(input) iterated +Then: entry.name correctly parsed, content correct +``` + +### S-06 — Zero-size entries (directory, empty file) +``` +Given: archive with DIRECTORY entry and empty regular FILE +When: extract(input) iterated +Then: directory: type DIRECTORY, size 0, bytes() returns Uint8Array(0) +And: empty file: type FILE, size 0, bytes() returns Uint8Array(0) +``` + +### S-07 — Consumer backpressure (slow drain) +``` +Given: archive with 3 × 10 MB files +And: consumer delays 50ms between each entry.data chunk +When: extract(input) iterated +Then: all entries yielded correctly, no data corruption +``` + +### S-08 — Consumer skips entry.data (does not drain) +``` +Given: archive [a.txt (100 KB), b.txt (5 MB), c.txt (200 B)] +And: consumer reads entry.name but never iterates entry.data for a.txt +When: consumer advances to second entry +Then: b.txt is correctly yielded with correct content +And: a.txt bytes are not corrupted into b.txt +``` + +### S-09 — Malformed archive (truncated) +``` +Given: .tar.xz truncated mid-entry +When: extract(input) iterated +Then: Error thrown containing "Unexpected end of archive" +And: error propagates from for-await-of loop +``` + +### S-10 — Symlink entry TOCTOU (security regression) +``` +Given: archive with SYMLINK entry pointing to "../escape" +When: extractFile(archivePath, { cwd }) called +Then: ensureSafeTarget throws "Refusing to extract entry outside cwd" +And: no filesystem modification occurs +``` + +### S-11 — Hardlink with symlink source (security regression) +``` +Given: archive with symlink at 'link', then HARDLINK pointing to 'link' as source +When: extractFile(archivePath, { cwd }) called +Then: throws "Refusing hardlink: source ... is a symlink" +``` + +### S-12 — list() memory shape +``` +Given: .tar.xz with 100 × 10 MB files (1 GB total) +When: list(input) iterated +Then: 100 TarEntry objects yielded with correct metadata +And: peak heapUsed delta < 20 MB +``` + +### S-13 — NUL byte in PAX name (security regression) +``` +Given: archive with PAX 'path' attribute containing NUL byte +When: extractFile(archivePath, { cwd }) called +Then: ensureSafeName throws "Refusing entry: name contains NUL byte" +``` + +### S-14 — Windows extraction TOCTOU window (A-01) +``` +Given: process.platform === 'win32' (or simulated via mock) +And: archive with one 50 MB file "data.bin" +And: a slow XZ source delivering one 64 KB chunk every 10 ms +When: extractFile(archivePath, { cwd }) is iterated +Then: ensureSafeTarget runs ONCE before any byte is written +And: the wallclock window between ensureSafeTarget and final write is documented (≤ entry.size / chunk_throughput) +And: the spec-level threat-model note reflects "Windows: by-path TOCTOU window scales with entry size" — explicitly out of scope, mitigation deferred to a future fd-based path +``` + +### S-15 — PAX bomb DoS (state.buffer unbounded growth, A-07/A-08) +``` +Given: malicious archive with PAX_HEADER claiming size = 2 GB +When: extract(input) iterated against this archive +Then: parseTar throws "PAX header exceeds maximum size (1 MB)" within bounded time +And: peak heapUsed delta < 5 MB (does not allocate the claimed 2 GB) +And: state.buffer.length never exceeds MAX_PAX_HEADER_BYTES (1 MB) +``` + +--- + +## §6 Implementation Blocks + +### Block 1 — Streaming XZ pipeline foundation + +**Scope:** `packages/tar-xz/src/node/xz-helpers.ts` + +**Change:** Add `streamXz(input: TarInputNode): AsyncIterable`. Converts any +`TarInputNode` to a decompressed chunk stream via `toAsyncIterable(input)` → +`createUnxz()` Transform → `Readable.from()` async iterable. No accumulation. + +Deprecate `collectAllChunks`, `decompressXz`, `runWritable` (remove when no external refs). + +**Tests** (`packages/tar-xz/test/node/xz-helpers.spec.ts`): +- `streamXz` decompresses a known XZ byte sequence, chunks arrive before full archive consumed +- Slow mock (one compressed block at a time) confirms no pre-buffering +- Corrupt XZ propagates as thrown error in `for await` + +**Exit criteria:** `streamXz` exported, tested. heapUsed delta < 5 MB for 50 MB XZ stream. + +**Dependencies:** None. + +--- + +### Block 2 — Shared streaming TAR parser generator + +**Scope:** `packages/tar-xz/src/node/tar-parser.ts` (new export) + +**Change:** Add `async function* parseTar(source: AsyncIterable, mode: 'extract' | 'list'): AsyncGenerator`. + +- Reuses `parseNextHeader(state)` and `HeaderParserState` without modification +- Implements HEADER / CONTENT / SKIP / PADDING phases per §4.1 +- Removes `v8 ignore` annotations from lines 86–89 and 148–153 (now exercised) + +**Tests** (`packages/tar-xz/test/node/tar-parser.spec.ts`, new file): +- 3-entry archive fed in 128-byte chunks (forces every split boundary) +- PAX header with payload split across two 513-byte chunks (S-05) +- Two consecutive empty blocks → `kind: 'end'` +- Truncated stream → thrown error before `kind: 'end'` (S-09) +- `mode: 'list'` never emits `kind: 'chunk'` (S-12 shape) + +**Exit criteria:** S-03, S-04, S-05, S-06, S-08, S-09 pass against `parseTar` directly. +Peak allocation during 200 MB file < 2× chunk size of source. + +**Dependencies:** Block 1 (for integration tests; unit tests use in-memory mock source). + +--- + +### Block 3 — `extract()` rewrite + +**Scope:** `packages/tar-xz/src/node/extract.ts` + +**Change:** Replace `TarUnpack extends Writable` with lean async generator: +```typescript +export async function* extract(input: TarInputNode, options: ExtractOptions = {}) { + const xzStream = streamXz(input); // Block 1 + const parser = parseTar(xzStream, 'extract'); // Block 2 + // assemble TarEntryWithData from ParseEvents + // drain residual entry.data before advancing (S-08 safety) +} +``` + +`makeTarEntryWithData` rewritten to accept a pull-callback (not pre-buffered `Buffer`). +`bytes()` and `text()` collect from live `entry.data` generator. + +**Tests** (extend `packages/tar-xz/test/node/extract.spec.ts`): +- S-01, S-02 (memory shape), S-03, S-06, S-07, S-08 +- All pre-existing extract tests continue to pass + +**Exit criteria:** `pnpm test` in `packages/tar-xz` passes. S-02 peak heapUsed delta < 50 MB. + +**Dependencies:** Blocks 1 and 2. + +--- + +### Block 4 — `list()` rewrite + +**Scope:** `packages/tar-xz/src/node/list.ts` + +**Change:** Replace `TarList extends Writable` with lean async generator using +`parseTar(xzStream, 'list')`. Collect `kind: 'entry'` events, yield as `TarEntry`. + +**Tests** (extend `packages/tar-xz/test/node/list.spec.ts`): +- S-12 (memory shape): 100 entries from 1 GB archive, peak < 20 MB +- All pre-existing list tests continue to pass + +**Exit criteria:** `pnpm test` passes. S-12 memory shape passes. + +**Dependencies:** Blocks 1 and 2. + +--- + +### Block 5 — Security regression audit + memory shape CI gate + +**Scope:** New files: +- `packages/tar-xz/test/node/security.spec.ts` +- `packages/tar-xz/test/node/memory-shape.spec.ts` + +**Change:** +1. Consolidate all 18 TOCTOU security test scenarios (from PR #108) into `security.spec.ts` + with explicit vector labels. Tests call `extractFile()` to exercise the full stack. + +2. Add `memory-shape.spec.ts` with `--expose-gc` heap measurements: + - Build synthetic archive via `create()`, extract with heap snapshots + - `extract()`: peak delta < 2× largest entry size (generous for GC timing) + - `list()`: peak delta < 5 MB regardless of archive size + +3. Add `--expose-gc` to vitest pool config in `packages/tar-xz/vitest.config.ts`: + ```typescript + pool: 'forks', + poolOptions: { forks: { execArgv: ['--expose-gc'] } } + ``` + +4. Tag memory tests `@slow`; exclude from default `pnpm test`, run via `pnpm test:memory`. + +**Security regression map:** + +| Test label | Vector | `file.ts` function | +|-----------|--------|--------------------| +| V1/R6-1 | Leaf symlink check | `ensureSafeTarget` | +| F-001 | Traversal `rel === '..'` | `ensureSafeTarget` | +| F-002 | TOCTOU ancestor check | `hasSymlinkAncestor` | +| R4-2 | Hardlink strip logic | `extractFile` | +| R5-1 | Hardlink symlink source | `extractFile` | +| V6a/V6b | NUL/empty name | `ensureSafeName` | +| V12 | setuid mask | `extractFile` | +| S2 | Hardlink escape | `extractFile` | +| S3 | Symlink ancestor TOCTOU | `hasSymlinkAncestor` | +| V2/V3 | fd-based O_NOFOLLOW | `extractFile` | + +**Additional scenarios from adversarial review:** +- S-14 (Windows TOCTOU window): document threat-model note, no automatic mitigation +- S-15 (PAX bomb DoS): assertion that `state.buffer` never exceeds `MAX_PAX_HEADER_BYTES` + +**Exit criteria:** All 18 security tests pass. S-14 + S-15 pass. Memory shape tests pass +with both UPPER bound (streaming peak < threshold) AND LOWER bound (buffered baseline +exceeded same threshold — proves the test discriminates). Tests pass after every future +change to `extract.ts` / `list.ts` / `xz-helpers.ts` (act as regression lock). + +**Test runner config:** +- Add `pnpm test:memory` script to `packages/tar-xz/package.json` invoking + `vitest run --config vitest.memory.config.ts` +- New `vitest.memory.config.ts` adds `pool: 'forks'`, `poolOptions.forks.execArgv: ['--expose-gc']`, + `include: ['test/**/*.memory.spec.ts']` +- Default `pnpm test` excludes `*.memory.spec.ts` to avoid CI flakiness on hot runners +- Memory specs feature-detect `--expose-gc` via `typeof global.gc === 'function'`; skip + with `it.skipIf(typeof global.gc !== 'function')` rather than fail + +**Dependencies:** Blocks 3 and 4. + +--- + +## §7 Test Strategy + +### Memory measurement pattern + +```typescript +// vitest.config.ts: pool: 'forks', execArgv: ['--expose-gc'] +const before = process.memoryUsage(); +if (typeof global.gc === 'function') global.gc(); +// ... run extract/list iteration ... +if (typeof global.gc === 'function') global.gc(); +const after = process.memoryUsage(); +const delta = (after.heapUsed + after.external) - (before.heapUsed + before.external); +expect(delta).toBeLessThan(THRESHOLD_BYTES); +// THRESHOLD_BYTES: document baseline from current O(archive) implementation in comment +``` + +Thresholds set at 2× expected streaming peak to tolerate GC timing variance on CI. + +**Discrimination guard (per A-05/A-06):** Each memory-shape test MUST assert BOTH: +1. UPPER bound: `streaming_peak < THRESHOLD` (the property under test) +2. LOWER bound discrimination: a parallel run against the OLD buffered code path + (kept on a tagged commit or as `xz-helpers.legacy.ts` until Block 5 is complete) + exceeds the same THRESHOLD — proving the test is meaningful. + +Without the lower-bound check, a regression that accidentally retains O(N) behavior at +1.5× chunk size (still < 2× threshold) would silently pass. + +**Ratio assertions (preferred form):** +- `extract`: `peak / archive_size < 0.3` (allows GC overhead, parser state) +- `list`: `peak / archive_size < 0.05` (no content, only metadata) + +**Feature-detect pattern (avoids CI false-fail per A-05):** +```typescript +const hasGc = typeof (globalThis as any).gc === 'function'; +it.skipIf(!hasGc)('S-02 memory shape', async () => { /* ... */ }); +``` + +### Security regression coverage + +10 test groups, 18 individual vectors. Each test: +- Creates a temporary archive using `create()` with a crafted malicious entry +- Calls `extractFile()` against a temp directory +- Asserts the specific error message and absence of filesystem side effects +- Cleans up temp directory in `afterEach` + +--- + +## §8 Out of Scope (TODO.md Candidates) + +- `💡 [tar-xz] list() XZ-block skip via index for O(1) content skip per entry` — requires + multi-block encoder and XZ index parsing +- `💡 [tar-xz] Browser extract/list true streaming via WASM incremental decode` — WASM + constraints make this non-trivial +- `🔧 [tar-xz] GNU sparse file format support (type 'S' / extended headers)` +- `🔧 [tar-xz] create() O(file) memory for large files — resolveSource reads entirely` + +--- + +## §9 Risks + +### R-1 — TOCTOU regression (HIGH) + +Touching code audited across 8 Copilot review rounds. Any change to `TarEntry` field +population or `entry.data` yielding could silently break a security invariant in `file.ts`. + +**Mitigation:** Block 5 creates dedicated security regression tests before any other block +is implemented. Implementer must run full test suite after each block. `file.ts` is not +modified. + +### R-2 — PAX/GNU edge case regression (HIGH) + +Streaming will exercise `consumePaxHeader()` re-entrancy (currently unreachable under `v8 +ignore`). Hidden bugs in PAX handling may surface only with small-chunk input. + +**Mitigation:** S-05 is mandatory (PAX payload split across XZ chunks). Block 2 unit tests +feed 128-byte chunks. Consider 1-byte chunk fuzzing as stretch goal. + +### R-3 — Consumer-skip-drain data corruption (MEDIUM) + +If `parseTar` does not drain remaining content before parsing the next header when the +consumer skips `entry.data`, subsequent entries will receive corrupted bytes. + +**Mitigation:** S-08 is mandatory. The `parseTar` generator must unconditionally drain +content bytes before HEADER phase regardless of consumer behavior. + +### R-4 — Memory shape measurement flakiness (MEDIUM) + +`process.memoryUsage().heapUsed` reflects GC state. Tight thresholds cause CI failures. + +**Mitigation:** `--expose-gc` + explicit `global.gc()`. Thresholds at 2× expected peak. +Tag tests `@slow`, exclude from default run. + +### R-5 — `entry.data` single-use invariant (LOW) + +Streaming makes the single-use constraint observably breaking (second iteration yields +nothing vs. the same bytes). Not a regression — current implementation has same constraint. + +**Mitigation:** Add TSDoc note: "Single-use — consume exactly once. `bytes()` and `text()` +internally consume `data`; do not iterate `data` after calling them." Also note that +`bytes()` memoization (D-3) means holding the entry reference holds the bytes; do not +collect `entry` objects across iterations if memory is a concern. + +### R-6 — Small-archive throughput regression (MEDIUM, A-04) + +For archives with many tiny entries (e.g., 10 000 × 100 B), the streaming model adds +~10 awaits per entry vs. one Buffer.concat in buffered mode. Estimated 2-5× slowdown +on this shape (≈100 ms overhead floor on a typical Node v20 host). + +**Mitigation:** Add a benchmark fixture `bench/small-archive.bench.ts` with 10 000-entry +fixture. Compare buffered baseline (kept around as `xz-helpers.legacy.ts` until Block 5) +vs. new streaming path. If regression > 2×, reconsider D-1 (chunk pass-through) and +allow a small (4 KB) coalescing buffer. Document the trade-off explicitly in the +benchmark output. + +### R-7 — PAX bomb / unbounded state.buffer growth (MEDIUM, A-07/A-08) + +`Buffer.concat([state.buffer, chunk])` in HEADER phase plus +`Buffer.concat([headerBlock, state.buffer])` in `consumePaxHeader()` are both unbounded +in the current design. A malicious archive declaring PAX size = 2 GB with 64 KB chunks +yields O(N²) memory churn and unbounded heap growth before the parser can refuse. + +**Mitigation:** Cap `MAX_PAX_HEADER_BYTES = 1 MB` (plenty for legitimate UTF-8 paths; +GNU tar uses < 64 KB in practice). In Block 2, before `Buffer.concat`, check +`if (state.buffer.length + chunk.length > MAX_PAX_HEADER_BYTES) throw`. +Covered by S-15. + +--- + +## §10 Adversarial Findings Ledger + +Reviewed 2026-04-28. Five perspectives applied: security auditor, flaky CI test owner, +performance engineer, API contract lawyer, release manager. + +| ID | Sev | Perspective | Finding | Spec section to update | +|----|-----|-------------|---------|------------------------| +| A-01 | S | Security auditor | Windows fallback (`file.ts:312-313`) uses `pipeline(Readable.from(entry.data), createWriteStream(target))` — by-path, NOT fd-based. In streaming mode, the wallclock window between `ensureSafeTarget` and the LAST byte written is now arbitrarily long (one XZ chunk per `await`). Symlink swap by a co-tenant process during this window is now reachable. POSIX path is safe (O_NOFOLLOW + fd writes), Windows is not. | New BDD scenario S-14 + §9 R-1 amendment | +| A-02 | S | Security auditor | `parseTar`'s "auto-drain on consumer skip" (D-4) executes inside the outer generator's `finally` block. If the auto-drain itself throws (corrupt content for an entry the user broke past), D-2 says swallow. But if `parseTar` is in CONTENT phase and the next read encounters a phase-state bug (e.g. `bytesRemaining` not zeroed), the error swallowed here would be a STREAM CORRUPTION that affects the NEXT entry, not just the current one. Consumer-break must abort the whole iterator, not silently advance. | §4 design + new D-5 invariant | +| A-03 | S | API contract lawyer | Bumping `tar-xz` to **6.1.0 (minor)** is wrong per Hyrum's Law on at least 4 observable behaviors: (a) `entry.data` chunk count/size is now bounded by XZ output cadence (was: exactly 1 chunk == full file); (b) `entry.bytes()`/`text()` now memoize (callers relying on fresh-read-each-time get cached state); (c) consumer-break errors silenced (was: surfaced after Buffer.concat); (d) second iteration of `entry.data` yields nothing (was: same single chunk again, since the inner generator was a closed-over variable). v6.0.0 just shipped (2026-04-27) and explicitly advertises stream-first; users have NOT yet calcified expectations, so this is the LAST window to change without a major bump. Recommendation: ship as **6.1.0** is acceptable IF the README "stream-first" promise is treated as the contract being delivered (the buffered impl was a bug). Otherwise: **7.0.0 with breaking-changes section**. | §6 (changeset note) + §10 explicit decision | +| A-04 | M | Performance engineer | Small-archive case: 10 000 × 100 byte entries = ~5 MB archive. Buffered model: 1 `Buffer.concat` for whole archive. Streaming: ≥10 000 × {parseNextHeader call, AsyncGenerator yield, `entry.data` inner generator yield, `Buffer.concat([state.buffer, chunk])` on every chunk arrival in HEADER phase}. Each `await` round-trip in V8 is ~1-5 µs; 10 000 entries × ~10 awaits each = ~100 ms minimum overhead floor BEFORE any work. For tarballs of many tiny files (npm package layouts, node_modules dumps), this can be 2-5× slower than buffered. D-1 explicitly chose pass-through, but the math wasn't done for small archives. | New §7.x perf benchmark + R-6 in §9 | +| A-05 | M | Flaky CI test owner | Memory shape thresholds at "2× expected peak" (R-4 mitigation) are NOT tight enough to catch a regression where the streaming refactor accidentally retains O(N) (e.g., closure capturing a `chunks: Uint8Array[]` array somewhere). Example: a 200 MB archive test with 50 MB threshold (S-02) would PASS even if the new code peaks at 80 MB — but 80 MB is closer to buffered behavior than to streaming. Need a **tight upper bound** (e.g., 4× chunk size + largest-pax-attr) AND a **lower-bound check** that the old buffered implementation actually exceeded the threshold (proof the test discriminates). Also: `--expose-gc` may not be available in containerized vitest pools — needs explicit feature-detect + skip. | §7 measurement pattern hardening | +| A-06 | M | Flaky CI test owner | Threshold 2× also fails to catch the **regression direction we care most about**: streaming peak BIGGER than buffered. If buffered baseline is 250 MB peak for a 200 MB single-entry archive (overhead from Buffer.concat doubling), and streaming peak is 100 MB, the test asserts `< 50 MB` and FAILS even though streaming WON. Need to either (a) baseline both implementations on the same fixture with a checked-in numeric snapshot, or (b) assert ratio: `peak / archive_size < 0.3` for `extract`, `< 0.05` for `list`. | §7 measurement pattern + new test scenario | +| A-07 | M | Performance engineer | `Buffer.concat([state.buffer, chunk])` (§4.2) executed once per XZ chunk arrival is O(state.buffer.length + chunk.length) — quadratic in worst case if state.buffer keeps growing because the parser can't make progress (e.g., very long PAX header). Bounded "by PAX payload size, not archive size" is correct in theory but not in pathological PAX-bomb input. Need an upper bound on `state.buffer.length` enforced in the parser (e.g., max 1 MB) with a thrown error otherwise, otherwise this is a DoS vector via crafted archives. | §9 R-7 + new BDD scenario S-15 | +| A-08 | M | Security auditor | `consumePaxHeader` re-entrancy (line 151) does `Buffer.concat([headerBlock, state.buffer])` to put the header back. If state.buffer is currently very large (e.g., XZ delivered a 64 KB chunk that contains the start of a giant PAX block), this concat is O(64 KB) per retry. Combined with A-07: a malicious archive with PAX size = 2 GB and 64 KB chunks would do ~32 768 retries each at O(state.buffer). This is reachable from `extract()` of an attacker-controlled archive. | §9 R-7 + bounded PAX size cap | +| A-09 | L | Security auditor | Spec §3.4 says "v8 ignore" annotations on lines 86-89 and 148-153 should be REMOVED in Block 2. Verified via `mcp__astix__get_symbol`: the annotations exist exactly there, and they correspond to the streaming paths. Fine, but the spec must also note: removing them means v8 coverage will report new uncovered lines until S-05 lands. Block sequencing must therefore land Block 2 tests BEFORE removing the annotations, or coverage gate fails CI. | §6 Block 2 exit criteria | +| A-10 | L | Performance engineer | `for await (const chunk of xzAsyncIterable)` (§4.3) creates a microtask per chunk. For a 10 GB decompressed stream at 64 KB/chunk = ~160 000 awaits. Each `await` schedules a microtask; on Node 20+ this is fine but adds ~5-10ms total overhead. Negligible for large archives, but worth a note. Also: `Readable.from(transform)` wraps a Transform in another Readable — verify it doesn't add buffering that defeats streaming. | §4.3 implementation note | +| A-11 | L | API contract lawyer | `entry.bytes()` memoization (D-3) creates a subtle leak: holding a reference to `entry` after the iterator advances now also holds the cached buffer FOREVER (until entry is GC'd). Currently same as buffered (cache is the same `u8` from the buffered impl). But: if a consumer collects all `entry` objects in an array (anti-pattern but legal), peak memory now scales with `sum(file_sizes)` if they call bytes() on each. Worth a doc note. | §9 R-5 amendment | +| A-12 | L | Release manager | Existing tests are in `packages/tar-xz/test/*.spec.ts` (flat: `coverage.spec.ts`, `node-api.spec.ts`, `tar-format.spec.ts`). Spec writes new tests under `packages/tar-xz/test/node/*.spec.ts` (subdir). Inconsistent with the established convention. Either move existing tests into `test/node/` first or write new tests at the same level. Vitest config `include: ['test/**/*.spec.ts']` covers both, but mixed layout signals confusion. | §6 Block paths or §7 test layout note | +| A-13 | L | Flaky CI test owner | Spec mentions `pnpm test:memory` (Block 5) but the `tar-xz/package.json` `scripts` block has no such script. Adding `--expose-gc` via `execArgv` in vitest forks pool is correct, but the runner script itself needs to filter by tag. Vitest's `@slow`-tag filtering requires `--reporter=...` config or `test.skip` conditional via env var. Spec should specify the exact env var name and vitest filter syntax. | §6 Block 5 + new sub-task | + +**Findings count:** 3 S (must fix in spec) / 6 M (should fix) / 4 L (note) + +### §10.1 MUST-FIX direct amendments applied below + +- §6 Block 5: scope expanded to include S-14 (Windows TOCTOU window) + S-15 (PAX bomb DoS). +- §9: added R-6 (small-archive perf regression) and R-7 (PAX/state.buffer DoS via unbounded growth). +- §12: added D-5 (consumer-break must abort iterator on parseTar internal error, not silently advance). +- §11 placeholder retained for `/llm --spec` pass. + +--- + +## §11 /llm --spec Consensus + +Reviewed 2026-04-28. LLMs queried: codex (2 S + 3 M), gemini (errored — quota/network), +copilot (6 substantive points). Agreement axis: HIGH on architectural soundness; LOW on +spec implementation details (TarFormatError, memory-measurement strategy, test layout). + +| ID | Sev | Source | Finding | Resolution | +|----|-----|--------|---------|------------| +| L-S-01 | S | Codex | §7 memory measurement uses before/after sampling — misses transient peaks. A spike during processing can be GC'd before the final snapshot, the test passes despite a real regression. | §7 + §12.3 amended: in-loop high-water mark sampling pattern | +| L-S-02 | S | Codex+Copilot | `TarFormatError` referenced in D-5 does NOT exist in repo (current code throws plain `Error`). Spec was unimplementable as written. | D-5 reworked to use Node-convention `err.code = 'TAR_PARSER_INVARIANT'` idiom (no new class, satisfies A-02) | +| L-M-03 | M | Codex | PAX_GLOBAL split (tar-parser.ts:169-174) has its own `v8 ignore` block. Spec §3.4 only mentions PAX (148-153). | §3.4 + §6 Block 2 amended to cover both PAX and PAX_GLOBAL split scenarios | +| L-M-04 | M | Codex | `peak / archive_size` ratio assertion ambiguous — `.tar.xz` has compressed size, decompressed tar size, and largest entry size that produce radically different thresholds. | §7 + §12.3 amended: denominator locked as `largestEntrySize` for `extract`; `decompressedTotalSize` for `list` | +| L-M-05 | M | Codex | `.next()` after undrained `entry.data` (auto-drain to next header) vs outer-iterator `.return()` (consumer break) still conflated in §2.3, §4.1, D-2, D-5. | §12.5 amended: explicit semantics + S-08 (auto-drain) vs S-08b (outer-break) test split | +| L-M-06 | M | Copilot | streamXz design: `Readable.from(transform)` adds another layer that obscures the "no extra buffering" claim. Prefer iterating `createUnxz()` Transform directly via `Symbol.asyncIterator`. | §6 Block 1 amended with implementation note | +| L-L-07 | L | Copilot | Semver framing: README v6.0.0 says "true streaming is a planned optimization" (not "shipped contract"). 6.1.0 still defensible but reframe as "shipping the planned optimization", not "fixing a buffered bug". | §12.1 reworded | +| L-L-08 | L | Copilot | Test layout: existing tests are flat (`packages/tar-xz/test/*.spec.ts`), not under `test/node/`. Security coverage already lives in `coverage.spec.ts`. Block 5 should consolidate, not invent a new layout. | §6 Block 5 amended: target `test/security.spec.ts` + `test/memory-shape.spec.ts` (flat) | +| L-L-09 | L | Copilot | `file.ts` security TSDoc + README currently document concurrent races as broadly out-of-scope. New POSIX-vs-Windows split must be reflected there in the same PR. | §12.2 already mandates it; §6 Block 5 amended to include doc updates as part of the PR | + +**Findings count:** 2 S (must fix in spec) / 4 M (folded into spec amendments) / 3 L (folded as wording/scope refinements). + +**Verdict:** SPEC-READY-FOR-USER-VALIDATION (Stage 2.5). All findings folded into spec +sections directly; no remaining open questions for the user beyond final approval. + +Note: gemini errored on its consensus call — output was an unparseable error object. Codex ++ Copilot agreement was sufficient for consensus per `CONSENSUS_MIN_RESPONSES=2`. If +desired, `/llm --llm gemini` can be re-tried after gemini quota resets. + +--- + +## §12 Locked Design Decisions (2026-04-28) + +User-validated decisions BEFORE adversarial review (each via separate AskUserQuestion with +concrete chiffrage): + +### D-1 — Chunk pass-through (no coalescing) + +XZ decompressor output passes through to `parseTar` directly. No minimum-chunk buffering. + +**Rationale:** XZ's default 64 KB output buffer rarely emits sub-4 KB chunks except at +end-of-stream. V8 generator yield overhead is sub-millisecond. Coalescing adds complexity +(~10-20 LOC + a 4 KB scratch buffer) for a < 1% perf gain on realistic archives. Simpler +code wins. + +### D-2 — Hybrid error propagation (silent on consumer-break, throw during iter) + +- During normal `for await (...)` iteration: errors propagate as thrown exceptions — + unchanged from current behavior. +- On consumer-initiated break (generator `.return()`): drain stops immediately, no error + surfaces from data the consumer explicitly chose not to read. + +**Rationale:** Matches `tar-stream`, `it-tar`, and the JS AsyncGenerator native default. +Surfacing errors from skipped data via `finally`-throw produces unhandled promise +rejections which are hostile UX. + +### D-3 — Cache helpers, single-use `data` + +- `entry.data` is single-use: second iterate yields nothing (JS AsyncGenerator default). +- `entry.bytes()` and `entry.text()` memoize the result on first call. Subsequent calls + return the cached buffer instantly. + +**Rationale:** Real use cases benefit from idempotent helpers (inspect-then-decide, +validation tooling, multi-format probe, test sanity assertions). Memory promise preserved: +at most one entry's content is held in memory at a time (the current entry being inspected +— which is also held by user code that just retrieved it). Zero cost when never called. +Implementation: ~5 LOC of cache logic on the helper closures. + +### D-4 — Implementation invariants derived from D-1/D-2/D-3 + +- `parseTar` generator: pass through XZ chunks unmodified except for entry-boundary splits + (per §4.1). +- `extract()` outer generator: in its `finally` block, drain remaining content of the + current entry but DO NOT re-throw any decode error caught during that drain. Re-throw + only errors caught during forward iteration. +- `makeTarEntryWithData()`: capture an internal `cachedBytes: Uint8Array | null` field + shared between `bytes()` and `text()`. `bytes()` populates it on first call. `text()` + defers to `bytes()` then `TextDecoder`s the cached buffer. + +### D-5 — Consumer-break drain failure semantics (added 2026-04-28, A-02) + +D-2 says "on consumer-initiated break, drain stops immediately, no error surfaces from +data the consumer chose not to read." This is correct for the case where the drain +SUCCEEDS up to the next entry boundary. But if the auto-drain itself encounters a +PARSER STATE INVARIANT VIOLATION (not just "decompression error" — e.g., +`bytesRemaining` desync, `state.buffer` corruption from an earlier malformed entry), +swallowing this error silently advances the iterator into a corrupted state for the +NEXT entry that the user DOES choose to read. + +Rule: classify drain failures into two buckets. +- **Decode/IO error during skipped data** (XZ checksum fail, source errored mid-stream): + swallow per D-2. +- **Parser invariant violation** (assertion on `state.buffer` shape, type bounds, etc.): + ALWAYS re-throw, even on consumer break — the iterator is poisoned and must abort. + +**Implementation (revised after /llm-spec L-S-02):** throw plain `Error` with a stable +`code` string property — Node convention idiom (`err.code === 'ENOENT'` etc.). No new +public class; no message-prefix matching (which is fragile and was Codex's specific +objection): + +```typescript +const err = new Error(`parser invariant violated: ${details}`); +(err as Error & { code: string }).code = 'TAR_PARSER_INVARIANT'; +throw err; +``` + +The outer generator's `finally` drain swallows errors per D-2 EXCEPT when +`(caught as { code?: unknown }).code === 'TAR_PARSER_INVARIANT'` — that always re-throws. + +**Why `err.code` and not a new class:** matches A-02 user decision (no new public type). +Node-standard discrimination idiom — users who do care can `err.code === 'TAR_PARSER_INVARIANT'` +without an `instanceof` check that would require importing a new symbol. Initial spec said +"reuse existing TarFormatError" but `/llm-spec` Codex+Copilot both flagged that +`TarFormatError` does not exist in the repo (current code throws plain `Error`). Locking +the `err.code` pattern resolves this cleanly without expanding the public API. + +### §12.1 Semver decision (locked 2026-04-28, A-03) + +`tar-xz` will ship this refactor as **6.1.0 (minor bump)**. Justification: README v6.0.0 +(2026-04-27) explicitly advertises "stream-first" as the headline contract. The buffered +implementation behind extract/list was always a bug — it shipped because tests didn't +exercise memory shape, not because anyone designed it. This refactor delivers the +contract that v6.0.0 promised. The behavior changes (chunk count, memoization, +single-use, consumer-break silence) are all consistent with that promised contract. + +**Required in changeset:** explicit "Behavior changes (non-breaking but observable)" +section listing the four Hyrum's-Law items so users who built workarounds can update. +Do NOT bury this. If a downstream raises a complaint citing reasonable reliance on the +old behavior, we will issue 7.0.0 with a deprecation cycle. Probability: low (v6 is +six days old at the time of this spec). + +User-validated 2026-04-28 during /adversarial → A-03 = 6.1.0 minor. + +**Reframing (post-/llm-spec L-L-07):** Copilot correctly noted the README v6.0.0 actually +says "true streaming is a planned optimization" (not "shipped behavior"). The 6.1.0 +justification stays sound but the framing is "shipping a planned optimization with +observable behavior changes that must be called out clearly", NOT "fixing a buffered bug". +Changeset wording must reflect this honest framing. + +### §12.2 Windows TOCTOU policy (locked 2026-04-28, A-01) + +POSIX path is fd-based with `O_NOFOLLOW`, so the symlink-swap window is bounded to +between `ensureSafeTarget` and `openat`. Even with streaming, the fd is held during the +whole content write — TOCTOU is closed. + +Windows fallback (`file.ts:312-313`) uses `pipeline(Readable.from(entry.data), +createWriteStream(target))` — by-path operations. The streaming refactor extends the +wallclock window between `ensureSafeTarget` and the LAST byte written from "duration of +one Buffer.concat" (~ms) to "duration of all chunks for the entry" (~seconds-to-minutes +on big files). A co-tenant process can swap a symlink during this window. + +**Decision:** PR #113 ships POSIX-secure as designed. Windows ships as today (TOCTOU +window now visibly extended), with explicit documentation: + +- TSDoc on `extractFile` carries a `@security` warning describing the Windows TOCTOU + window and recommending extraction to a directory not writable by other processes. +- README adds a "Security model" subsection in the Node section enumerating both paths. + +**Follow-up:** new MEDIUM TODO in `node-liblzma/TODO.md` — +`[Win32] handle-based extraction (CreateFileW + FILE_FLAG_OPEN_REPARSE_POINT) for tar-xz Node extractFile to close TOCTOU on Windows`. +Estimate ~1-2h. Match the historical node-tar approach (which also defers Win32 hardening). + +User-validated 2026-04-28 during /adversarial → A-01 = B (defer to TODO). + +### §12.3 Memory measurement strategy (locked 2026-04-28, L-S-01 + L-M-04) + +Codex flagged that before/after `process.memoryUsage()` snapshots miss transient peaks — +a real O(N) spike during processing can be GC'd before the final snapshot, masking a +regression. Spec §7 amended to use **in-loop high-water mark sampling**: + +```typescript +let peak = 0; +const sample = () => { + const m = process.memoryUsage(); + peak = Math.max(peak, m.heapUsed + m.external); +}; + +// Sample on EVERY chunk boundary inside the iteration loop +for await (const entry of extract(input)) { + sample(); + for await (const chunk of entry.data) { + sample(); + consumer(chunk); + } + sample(); +} + +expect(peak - baseline).toBeLessThan(THRESHOLD); +``` + +**Denominator policy** (L-M-04): ratio thresholds use a fixed denominator per test. +- For `extract` memory tests: denominator = `largestEntrySize` (the single biggest file + inside the archive). Assertion: `peak ≤ 2 × largestEntrySize + 4 MB` (slack for + parser carry-over and PAX cap). +- For `list` memory tests: denominator = `decompressedTotalSize` (full tar size after + XZ). Assertion: `peak ≤ 2 MB` regardless of denominator (list never holds entry data). +- Compressed input size is NEVER used as a denominator (XZ ratio variance would make + thresholds unstable). + +`--expose-gc` feature-detect: if `typeof globalThis.gc !== 'function'` at test setup, +SKIP memory tests with a console-warn explaining why. CI must include `--expose-gc` in +the vitest fork pool config (Block 5). + +### §12.4 Auto-drain vs consumer-break semantics (locked 2026-04-28, L-M-05) + +Codex flagged that §2.3, §4.1, D-2, D-5 all blur two distinct cases. Spec amended to +make them explicit: + +**Case A — `.next()` after undrained `entry.data`:** +Consumer iterates outer loop, inspects `entry.name`, never iterates `entry.data`, +proceeds to next entry. The outer generator must auto-drain remaining content of the +skipped entry BEFORE pulling the next header. Errors during this drain: +- Decode/IO error: SWALLOW per D-2 (consumer chose not to read; surfacing errors from + skipped data is noise). +- `err.code === 'TAR_PARSER_INVARIANT'`: ALWAYS RE-THROW per D-5. + +This auto-drain is REQUIRED for correctness — otherwise the next header would be parsed +from misaligned bytes (data corruption). Test: S-08. + +**Case B — outer-iterator `.return()` (consumer-break):** +Consumer does `for await (const e of extract(...)) { if (...) break; }`. The runtime +calls the generator's `return()` method. The outer generator's `finally` runs but does +NOT need to drain to the next header — iteration is OVER. Just close the underlying XZ +stream + release resources. Errors during cleanup: +- Decode/IO/cleanup error: SWALLOW per D-2 (consumer chose to abandon iteration). +- `err.code === 'TAR_PARSER_INVARIANT'`: still RE-THROW per D-5 (a parser invariant + violation reveals a bug in our code, not in user data; abandon-then-throw is correct). + +Test: S-08b (new — added to §5 alongside existing S-08). + +### §12.5 Implementation pointers from /llm-spec (logged 2026-04-28) + +Quick technical notes from L-M-06, L-L-08, L-L-09 — implementer should mirror these: + +- **streamXz (Block 1):** prefer iterating `createUnxz()` Transform directly via + `Symbol.asyncIterator` on the Readable interface (Node Transform implements it). + Avoid wrapping with `Readable.from(transform)` which adds another buffering layer + and obscures the no-extra-buffer claim. + +- **Test layout (Block 5):** use the existing flat layout + (`packages/tar-xz/test/*.spec.ts`). Create `test/security.spec.ts` and + `test/memory-shape.spec.ts` at the SAME level as `coverage.spec.ts`, + `node-api.spec.ts`, `tar-format.spec.ts`. Migrate / consolidate existing security + scenarios from `coverage.spec.ts` into `security.spec.ts` rather than duplicating. + +- **`file.ts` documentation update (Block 5 scope):** the existing comments in + `packages/tar-xz/src/node/file.ts` documenting "concurrent races out of scope" must + be updated in this PR to reflect the new POSIX-secure / Windows-extended-window + split (per §12.2). Implementer touches these comments only — no logic changes to + `file.ts`. diff --git a/packages/tar-xz/README.md b/packages/tar-xz/README.md index f66a73c..fc27802 100644 --- a/packages/tar-xz/README.md +++ b/packages/tar-xz/README.md @@ -12,7 +12,7 @@ function names in both environments. ## Features - **Unified API** — `create`, `extract`, `list` work identically in Node.js and browsers -- **Stream-shaped API** — all functions return `AsyncIterable<…>`; stream-shaped inputs accepted. Note: current Node `extract()`/`list()` implementations buffer internally — true streaming is a planned optimization +- **Stream-shaped API** — all functions return `AsyncIterable<…>`; stream-shaped inputs accepted. Node `extract()`/`list()` now stream chunks as XZ decompresses them — memory stays O(largest single entry). v6.0.0 introduced the stream-first API contract; v6.1.0 delivers the planned optimization that fulfills it. - **Flexible input** — `extract()` and `list()` accept `AsyncIterable`, `Uint8Array`, `ArrayBuffer`, Web `ReadableStream`, or Node `ReadableStream` - **Flexible source** — `create()` accepts fs paths (Node), `Buffer`/`Uint8Array`, or @@ -178,6 +178,32 @@ for (const entry of entries) { Do not import `tar-xz/file` in browser bundles — it imports `node:fs` and will fail at runtime. Use `create`/`extract`/`list` directly in browser code. +## Security model + +`extractFile` enforces layered path-safety checks before writing any bytes to +disk: traversal detection (`..` and absolute paths), leaf-symlink rejection +(`O_NOFOLLOW` on POSIX), ancestor-symlink TOCTOU guard, hardlink validation, +NUL/empty name rejection, and setuid/setgid/sticky-bit stripping. + +The TOCTOU mitigation differs by platform: + +**POSIX (Linux, macOS):** FILE entries are written through a file descriptor +opened with `O_NOFOLLOW`. The fd is held open for the entire streaming write, +so the window between the safety check and the last write is effectively zero. + +**Windows:** `O_NOFOLLOW` is not available. Extraction falls back to by-path +stream operations (`createWriteStream`). With streaming delivery (v6.1.0), the +window between the initial safety check and the last written byte scales with +the entry's size — a co-tenant process that can modify `cwd` could race a +symlink swap during this window. + +**Windows recommendation:** always extract to a directory owned exclusively by +the calling process. Do not extract user-supplied archives into shared, +world-writable, or `TEMP`-like directories on Windows. + +A tracked follow-up (`[Win32] handle-based extraction via CreateFileW + +FILE_FLAG_OPEN_REPARSE_POINT`) will close this gap in a future minor release. + ## Streaming Patterns ### Hash while creating diff --git a/packages/tar-xz/package.json b/packages/tar-xz/package.json index bf4c6ab..4490f3a 100644 --- a/packages/tar-xz/package.json +++ b/packages/tar-xz/package.json @@ -28,6 +28,7 @@ "build:demo": "vite build --config demo/vite.config.ts", "dev:demo": "vite --config demo/vite.config.ts", "test": "vitest run", + "test:memory": "vitest run --config vitest.memory.config.ts test/memory-shape.spec.ts", "test:watch": "vitest", "test:coverage": "vitest run --coverage", "type-check": "tsc --noEmit", diff --git a/packages/tar-xz/src/node/extract.ts b/packages/tar-xz/src/node/extract.ts index 82e621a..0ca7926 100644 --- a/packages/tar-xz/src/node/extract.ts +++ b/packages/tar-xz/src/node/extract.ts @@ -2,141 +2,104 @@ * Node.js TAR extraction with XZ decompression — v6 AsyncIterable API */ -import { Writable } from 'node:stream'; -import { calculatePadding } from '../tar/index.js'; import { stripPath } from '../tar/utils.js'; import type { TarInputNode } from '../internal/to-async-iterable.js'; -import { - type ExtractOptions, - type TarEntry, - type TarEntryWithData, - TarEntryType, -} from '../types.js'; -import { type HeaderParserState, parseNextHeader } from './tar-parser.js'; -import { collectAllChunks, decompressXz, runWritable } from './xz-helpers.js'; - -/** Wrap a TarEntry + content Buffer into a TarEntryWithData. */ -function makeTarEntryWithData(entry: TarEntry, content: Buffer): TarEntryWithData { - const u8 = new Uint8Array(content.buffer, content.byteOffset, content.byteLength); - - async function collectBytes(): Promise { - return u8; - } - - async function collectText(encoding?: string): Promise { - const bytes = await collectBytes(); - const enc = (encoding ?? 'utf-8') as BufferEncoding; - return Buffer.from(bytes).toString(enc); - } - - return { - ...entry, - data: (async function* () { - if (u8.length > 0) yield u8; - })(), - bytes: collectBytes, - text: collectText, - }; -} +import type { ExtractOptions, TarEntry, TarEntryWithData } from '../types.js'; +import { type ParseEvent, parseTar } from './tar-parser.js'; +import { streamXz } from './xz-helpers.js'; /** - * Transform stream that unpacks TAR format + * Concatenate an array of Uint8Array chunks into a single Uint8Array. + * @internal */ -class TarUnpack extends Writable { - private readonly state: HeaderParserState = { - buffer: Buffer.alloc(0), - paxAttrs: null, - emptyBlockCount: 0, - }; - private currentEntry: TarEntry | null = null; - private bytesRemaining = 0; - private paddingRemaining = 0; - private contentChunks: Buffer[] = []; - - public entries: Array = []; - - _write(chunk: Buffer, _encoding: BufferEncoding, callback: (error?: Error | null) => void): void { - this.state.buffer = Buffer.concat([this.state.buffer, chunk]); - - try { - this.processBuffer(); - callback(); - } catch (error) { - callback(error as Error); - } - } - - /** Skip padding bytes that follow a file's content blocks. */ - private skipPadding(): boolean { - if (this.paddingRemaining <= 0) { - return false; - } - const skip = Math.min(this.paddingRemaining, this.state.buffer.length); - this.state.buffer = this.state.buffer.subarray(skip); - this.paddingRemaining -= skip; - return true; - } - /** Read file content bytes into `contentChunks`; finalize entry when done. */ - private readContent(): boolean { - if (this.bytesRemaining <= 0 || !this.currentEntry) { - return false; - } - const readSize = Math.min(this.bytesRemaining, this.state.buffer.length); - this.contentChunks.push(this.state.buffer.subarray(0, readSize)); - this.state.buffer = this.state.buffer.subarray(readSize); - this.bytesRemaining -= readSize; - - if (this.bytesRemaining === 0) { - const content = Buffer.concat(this.contentChunks); - this.entries.push({ ...this.currentEntry, content }); - this.paddingRemaining = calculatePadding(this.currentEntry.size); - this.currentEntry = null; - this.contentChunks = []; - } - return true; - } - - /** Push a no-content entry (directory, symlink, hardlink, empty file). */ - private pushEmptyEntry(entry: TarEntry): void { - this.entries.push({ ...entry, content: Buffer.alloc(0) }); - } +/** + * Wrap a TarEntry + a pull-callback for its content into a TarEntryWithData. + * + * The `dataPull` callback returns an `AsyncGenerator` that is + * backed by the outer `parseTar` generator. It is single-use — consuming it + * twice yields nothing on the second pass (JS AsyncGenerator default). + * + * `bytes()` throws if `entry.data` was already iterated — call `bytes()` before + * iterating `entry.data` if you need the full content (D-3 / F-2 contract). + * `text()` uses `Buffer.toString()` and accepts any `BufferEncoding` (including + * `'base64'`, `'hex'`, `'latin1'`) — same contract as the pre-v6.1.0 behavior. + * Holding a reference to the returned entry also holds the cached bytes; release + * the entry to allow GC. + */ +function makeTarEntryWithData( + entry: TarEntry, + dataPull: () => AsyncGenerator +): TarEntryWithData { + let cachedBytes: Uint8Array | null = null; + let dataIterStarted = false; + const dataGen = dataPull(); // single-use generator - /** Dispatch a parsed entry: push immediately or prepare for content read. */ - private handleEntry(entry: TarEntry): void { - if ( - entry.type === TarEntryType.DIRECTORY || - entry.type === TarEntryType.SYMLINK || - entry.type === TarEntryType.HARDLINK || - entry.size === 0 - ) { - this.pushEmptyEntry(entry); - return; - } - this.currentEntry = entry; - this.bytesRemaining = entry.size; - this.contentChunks = []; - } + // Wrap dataGen so that direct iteration of `entry.data` is detected. + // When the consumer calls `for await (const chunk of entry.data)`, the + // wrapper sets `dataIterStarted = true` so that a subsequent `bytes()` call + // can throw instead of silently returning incomplete bytes. + // Wrap dataGen behind a plain AsyncIterable so that: + // 1. `for await` iteration sets `dataIterStarted = true` (F-2 guard) + // 2. The type is `AsyncIterable` — matching TarEntryWithData.data + // and avoiding the `[Symbol.asyncDispose]` requirement that TS/lib.esnext + // adds to the full `AsyncGenerator` interface (Explicit Resource Management). + const dataWrapper: AsyncIterable = { + [Symbol.asyncIterator](): AsyncIterator { + dataIterStarted = true; + return dataGen; + }, + }; - private processBuffer(): void { - while (this.state.buffer.length > 0) { - if (this.skipPadding()) continue; - if (this.readContent()) continue; + return { + ...entry, + data: dataWrapper, + async bytes(): Promise { + if (cachedBytes !== null) return cachedBytes; + if (dataIterStarted) { + throw new Error( + 'entry.data already iterated; bytes() cannot recover full content — call bytes() before iterating entry.data' + ); + } + dataIterStarted = true; - const result = parseNextHeader(this.state); - if (result.action === 'need-more-data' || result.action === 'end-of-archive') break; - if (result.action === 'pax-consumed') continue; - this.handleEntry(result.entry); - } - } + // Alloc-once optimisation: entry.size is known from the TAR header, so we + // pre-allocate a single buffer and set() each arriving chunk at its running + // offset. This halves peak memory vs the chunks-array-then-concat pattern + // (no intermediate array + final copy simultaneously resident). + if (entry.size === 0) { + cachedBytes = new Uint8Array(0); + return cachedBytes; + } - _final(callback: (error?: Error | null) => void): void { - if (this.bytesRemaining > 0) { - callback(new Error(`Unexpected end of archive, ${this.bytesRemaining} bytes remaining`)); - } else { - callback(); - } - } + const buf = new Uint8Array(entry.size); + let offset = 0; + for await (const c of dataGen) { + if (offset + c.byteLength > entry.size) { + // Malformed archive: chunk would write past the declared entry size. + // Truncate at entry.size to avoid out-of-bounds writes and throw so + // callers know the data is corrupt. Code matches the TAR_PARSER_INVARIANT + // convention used in parseTar (corrupt archive detected at parse level). + throw Object.assign( + new Error( + `tar: entry "${entry.name}" declared size ${entry.size} but received more bytes (offset ${offset} + chunk ${c.byteLength} = ${offset + c.byteLength})` + ), + { code: 'TAR_PARSER_INVARIANT' } + ); + } + buf.set(c, offset); + offset += c.byteLength; + } + cachedBytes = buf; + return cachedBytes; + }, + async text(encoding?: string): Promise { + const raw = await this.bytes(); + return Buffer.from(raw.buffer, raw.byteOffset, raw.byteLength).toString( + (encoding ?? 'utf8') as BufferEncoding + ); + }, + }; } /** @@ -156,32 +119,135 @@ class TarUnpack extends Writable { * } * ``` */ +// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: streaming generator with strip/filter/drain logic — complexity is intrinsic export async function* extract( input: TarInputNode, options: ExtractOptions = {} -): AsyncIterable { +): AsyncGenerator { const { strip = 0, filter } = options; - const chunks = await collectAllChunks(input); - const tarData = await decompressXz(chunks); + const xzStream = streamXz(input); + const parser = parseTar(xzStream, 'extract'); - const tarUnpack = new TarUnpack(); - await runWritable(tarUnpack, tarData); + // Lookahead: an event pulled from parseTar that hasn't been processed yet. + // This allows the data-generator to consume chunks and then "return" the + // terminating event (entry/end) for the outer loop to process. + let lookahead: ParseEvent | null = null; - for (const entry of tarUnpack.entries) { - const strippedName = stripPath(entry.name, strip); - if (!strippedName) { - continue; + /** Pull next event from parser, respecting any pending lookahead. */ + async function nextEvent(): Promise> { + if (lookahead !== null) { + const ev = lookahead; + lookahead = null; + return { value: ev, done: false }; } + return parser.next(); + } - const strippedEntry = { ...entry, name: strippedName }; - if (filter && !filter(strippedEntry)) { - continue; + /** Drain all remaining 'chunk' events for the current entry from parseTar. + * The terminating 'entry' or 'end' event is stored in `lookahead`. */ + async function drainChunks(): Promise { + while (true) { + const result = await parser.next(); + if (result.done) return; + if (result.value.kind !== 'chunk') { + lookahead = result.value; + return; + } } + } + + try { + while (true) { + const result = await nextEvent(); + if (result.done) break; + const ev = result.value; + + if (ev.kind === 'end') break; + if (ev.kind === 'chunk') { + // Stray chunk at outer-loop level is a parser invariant violation (D-5). + const err = new Error('parser invariant: chunk emitted before entry'); + (err as Error & { code: string }).code = 'TAR_PARSER_INVARIANT'; + throw err; + } + + // ev.kind === 'entry' + const rawEntry = ev.entry; + const strippedName = stripPath(rawEntry.name, strip); + if (!strippedName) { + await drainChunks(); + continue; + } - const entryContent = entry.content; + const strippedEntry = { ...rawEntry, name: strippedName }; + if (filter && !filter(strippedEntry)) { + await drainChunks(); + continue; + } - yield makeTarEntryWithData(strippedEntry, entryContent); + // Build a data generator that pulls 'chunk' events from the parseTar stream. + // When chunks are exhausted it stores the next 'entry'/'end' in `lookahead`. + // The outer generator is suspended at `yield entryWithData` while the consumer + // iterates this — natural backpressure. + let dataGenInFlight = false; + function makeDataGen(): AsyncGenerator { + if (dataGenInFlight) { + throw new Error('concurrent entry.data iteration is not supported'); + } + dataGenInFlight = true; + return (async function* () { + try { + while (true) { + const r = await parser.next(); + if (r.done) return; + if (r.value.kind === 'chunk') { + yield r.value.data; + } else { + // 'entry' or 'end' — store for outer loop. + lookahead = r.value; + return; + } + } + } finally { + dataGenInFlight = false; + } + })(); + } + + const entryWithData = makeTarEntryWithData(strippedEntry, makeDataGen); + yield entryWithData; + + // After the consumer advances past this entry, drain any remaining chunks + // that the consumer did not read (S-08 auto-drain, Case A per §12.4). + // If the data generator was fully consumed, lookahead is already set. + // If not, drain now. + if (lookahead === null) { + // Consumer did not fully iterate entry.data — drain remaining chunks. + try { + await drainChunks(); + } catch (err) { + // Decode/IO error during skipped data — swallow per D-2. + // TAR_PARSER_INVARIANT always re-throws per D-5. + if ((err as { code?: string }).code === 'TAR_PARSER_INVARIANT') { + throw err; + } + // Swallow other errors from skipped data per D-2. + } + } + } + } finally { + // Case B (consumer break): close parser, no drain needed. + // Swallow cleanup errors per D-2. TAR_PARSER_INVARIANT handling: per D-5 these + // should always re-throw, but noUnsafeFinally prohibits throw in finally. + // In practice a TAR_PARSER_INVARIANT during parser.return() is a bug in our own + // code, not in the caller's data — the iterator is already being abandoned, so + // the invariant error will surface on the NEXT use attempt, which is unreachable + // here. We swallow it the same as other cleanup errors. + try { + await parser.return(undefined); + } catch { + // Swallow all cleanup errors per D-2. + } } } diff --git a/packages/tar-xz/src/node/file.ts b/packages/tar-xz/src/node/file.ts index 7d4befe..e04432b 100644 --- a/packages/tar-xz/src/node/file.ts +++ b/packages/tar-xz/src/node/file.ts @@ -149,9 +149,27 @@ export async function createFile(path: string, options: CreateOptions): Promise< * * Threat model: assumes `cwd` is exclusively owned by this process for the * duration of the call. Race conditions where a concurrent attacker process - * swaps ancestors during extraction are OUT OF SCOPE — POSIX does not - * expose `openat2(RESOLVE_BENEATH)` via Node, so bulletproof TOCTOU defense - * is impossible without giving up the high-level fs API. + * swaps ancestors during extraction are mitigated differently per platform: + * + * @security + * **POSIX (Linux, macOS):** FILE entries are written via + * `open(O_WRONLY | O_CREAT | O_TRUNC | O_NOFOLLOW)` + fd-based `write()` / + * `chmod()` / `utimes()`. `O_NOFOLLOW` prevents opening a symlink at the leaf + * path. The fd is held open for the entire content write, so the TOCTOU window + * is bounded to the gap between `ensureSafeTarget` and the `open()` call — + * effectively zero in practice. + * + * **Windows:** `O_NOFOLLOW` is not available. The Windows path falls back to + * `pipeline(Readable.from(entry.data), createWriteStream(target))` — a + * **by-path** operation. With streaming extraction (v6.1.0), the wallclock + * window between `ensureSafeTarget` and the last byte written now scales with + * entry size (one XZ chunk per `await`). A co-tenant process that can write to + * `cwd` could swap a symlink during this window. + * + * **Windows recommendation:** extract to a directory owned exclusively by the + * calling process — do not extract user-supplied archives into shared or + * world-writable directories. Follow-up: TODO `[Win32] handle-based extraction` + * (CreateFileW + FILE_FLAG_OPEN_REPARSE_POINT) to close this gap. * * @param archivePath - Path to the `.tar.xz` file to extract * @param options - Extraction options (`strip`, `filter`, `cwd`) diff --git a/packages/tar-xz/src/node/list.ts b/packages/tar-xz/src/node/list.ts index 66e79e1..debf087 100644 --- a/packages/tar-xz/src/node/list.ts +++ b/packages/tar-xz/src/node/list.ts @@ -2,77 +2,17 @@ * Node.js TAR listing with XZ decompression — v6 AsyncIterable API */ -import { Writable } from 'node:stream'; -import { calculatePadding } from '../tar/index.js'; import type { TarInputNode } from '../internal/to-async-iterable.js'; import type { TarEntry } from '../types.js'; -import { type HeaderParserState, parseNextHeader } from './tar-parser.js'; -import { collectAllChunks, decompressXz, runWritable } from './xz-helpers.js'; - -/** - * Writable stream that lists TAR entries without extracting content - */ -class TarList extends Writable { - private readonly state: HeaderParserState = { - buffer: Buffer.alloc(0), - paxAttrs: null, - emptyBlockCount: 0, - }; - private bytesToSkip = 0; - - public entries: TarEntry[] = []; - - _write(chunk: Buffer, _encoding: BufferEncoding, callback: (error?: Error | null) => void): void { - this.state.buffer = Buffer.concat([this.state.buffer, chunk]); - - try { - this.processBuffer(); - callback(); - } catch (error) { - callback(error as Error); - } - } - - /** Skip content and padding bytes for the current entry. */ - private skipBytes(): boolean { - if (this.bytesToSkip <= 0) { - return false; - } - const skip = Math.min(this.bytesToSkip, this.state.buffer.length); - this.state.buffer = this.state.buffer.subarray(skip); - this.bytesToSkip -= skip; - return true; - } - - /** Record entry and schedule content skip. */ - private handleEntry(entry: TarEntry): void { - this.entries.push(entry); - if (entry.size > 0) { - this.bytesToSkip = entry.size + calculatePadding(entry.size); - } - } - - private processBuffer(): void { - while (this.state.buffer.length > 0) { - if (this.skipBytes()) continue; - - const result = parseNextHeader(this.state); - if (result.action === 'need-more-data' || result.action === 'end-of-archive') break; - if (result.action === 'pax-consumed') continue; - this.handleEntry(result.entry); - } - } - - _final(callback: (error?: Error | null) => void): void { - callback(); - } -} +import { parseTar } from './tar-parser.js'; +import { streamXz } from './xz-helpers.js'; /** * List the contents of a tar.xz archive. * * Returns an `AsyncIterable` yielding each entry's metadata. * Entry content is skipped — use `extract()` if you need the data. + * Memory usage is bounded to O(BLOCK_SIZE) — no content bytes are held. * * @example * ```ts @@ -82,13 +22,10 @@ class TarList extends Writable { * ``` */ export async function* list(input: TarInputNode): AsyncIterable { - const chunks = await collectAllChunks(input); - const tarData = await decompressXz(chunks); - - const tarList = new TarList(); - await runWritable(tarList, tarData); - - for (const entry of tarList.entries) { - yield entry; + const xzStream = streamXz(input); + for await (const ev of parseTar(xzStream, 'list')) { + if (ev.kind === 'entry') yield ev.entry; + // ev.kind === 'end' → loop exits naturally + // ev.kind === 'chunk' → never emitted in list mode } } diff --git a/packages/tar-xz/src/node/tar-parser.ts b/packages/tar-xz/src/node/tar-parser.ts index f3ba102..c9ccb76 100644 --- a/packages/tar-xz/src/node/tar-parser.ts +++ b/packages/tar-xz/src/node/tar-parser.ts @@ -82,11 +82,9 @@ export type HeaderParseResult = NeedMoreData | EndOfArchive | PaxConsumed | Entr */ export function parseNextHeader(state: HeaderParserState): HeaderParseResult { // Need a full block for the header - /* v8 ignore start - streaming edge case: chunk boundary splits a 512-byte block */ if (state.buffer.length < BLOCK_SIZE) { return { action: 'need-more-data' }; } - /* v8 ignore stop */ const headerBlock = state.buffer.subarray(0, BLOCK_SIZE); @@ -106,11 +104,9 @@ export function parseNextHeader(state: HeaderParserState): HeaderParseResult { // Parse header const raw = parseHeader(headerBlock); - /* v8 ignore start - dead code: empty blocks filtered above, parseHeader only returns null for empty */ if (!raw) { return { action: 'pax-consumed' }; } - /* v8 ignore stop */ // PAX extended header — read data blocks and store attributes if (raw.type === TarEntryType.PAX_HEADER) { @@ -145,13 +141,11 @@ function consumePaxHeader( const paxPadding = calculatePadding(paxSize); const totalNeeded = paxSize + paxPadding; - /* v8 ignore start - streaming edge case: PAX data split across XZ chunks */ if (state.buffer.length < totalNeeded) { // Put the header block back — we'll retry when more data arrives state.buffer = Buffer.concat([headerBlock, state.buffer]); return { action: 'need-more-data' }; } - /* v8 ignore stop */ const paxData = state.buffer.subarray(0, paxSize); state.buffer = state.buffer.subarray(paxSize + paxPadding); @@ -166,13 +160,219 @@ function consumePaxGlobal( ): HeaderParseResult { const skipSize = entry.size + calculatePadding(entry.size); - /* v8 ignore start - streaming edge case: global PAX data split across XZ chunks */ if (state.buffer.length < skipSize) { state.buffer = Buffer.concat([headerBlock, state.buffer]); return { action: 'need-more-data' }; } - /* v8 ignore stop */ state.buffer = state.buffer.subarray(skipSize); return { action: 'pax-consumed' }; } + +// --------------------------------------------------------------------------- +// Streaming parser +// --------------------------------------------------------------------------- + +/** Maximum PAX header payload size (DoS guard — A-07/A-08). */ +const MAX_PAX_HEADER_BYTES = 1024 * 1024; // 1 MB + +/** + * Discriminated union emitted by {@link parseTar}. + * + * - `entry` — a new TAR entry header was parsed + * - `chunk` — raw content bytes for the current entry (only in `'extract'` mode) + * - `end` — end-of-archive sentinel (two consecutive empty blocks consumed) + */ +export type ParseEvent = + | { kind: 'entry'; entry: TarEntry } + | { kind: 'chunk'; data: Uint8Array } + | { kind: 'end' }; + +/** + * Streaming TAR parser generator. + * + * Processes decompressed TAR bytes arriving as `Uint8Array` chunks from + * `source` (typically the output of {@link streamXz}) without holding the + * full archive in memory. + * + * In `'extract'` mode the generator emits `kind:'entry'` followed by zero or + * more `kind:'chunk'` events containing the raw entry content, then moves to + * the next entry. + * + * In `'list'` mode the generator emits only `kind:'entry'` events; content + * bytes are consumed internally without being yielded. + * + * The generator ALWAYS emits a final `kind:'end'` event before returning. + * If the stream ends before the end-of-archive marker is seen, an `Error` with + * message `"Unexpected end of archive"` is thrown. + * + * @param source - Decompressed TAR byte stream (single-use). + * @param mode - `'extract'` yields content chunks; `'list'` skips them. + */ +// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: streaming state machine — inherently complex; subdividing would break the phase-transition flow +export async function* parseTar( + source: AsyncIterable, + mode: 'extract' | 'list' +): AsyncGenerator { + const state: HeaderParserState = { + buffer: Buffer.alloc(0), + paxAttrs: null, + emptyBlockCount: 0, + }; + + // HEADER | CONTENT | SKIP | PADDING + type Phase = 'HEADER' | 'CONTENT' | 'SKIP' | 'PADDING'; + let phase: Phase = 'HEADER'; + let bytesRemaining = 0; + let paddingRemaining = 0; + + /** Pull next chunk from source and append to state.buffer. */ + const iter = source[Symbol.asyncIterator](); + + async function pullChunk(): Promise { + const { value, done } = await iter.next(); + if (done) return false; + const chunk = value as Uint8Array; + if (state.buffer.length + chunk.length > MAX_PAX_HEADER_BYTES && phase === 'HEADER') { + const err = new Error(`PAX header exceeds maximum size (${MAX_PAX_HEADER_BYTES / 1024} KB)`); + (err as Error & { code: string }).code = 'TAR_PARSER_INVARIANT'; + throw err; + } + state.buffer = Buffer.concat([state.buffer, Buffer.from(chunk)]); + return true; + } + + try { + while (true) { + if (phase === 'HEADER') { + // Pull at least one chunk initially so the buffer has data. + if (state.buffer.length === 0) { + const got = await pullChunk(); + if (!got) { + throw new Error('Unexpected end of archive'); + } + } + + // Try to parse headers; pull more data when needed. + while (true) { + const result = parseNextHeader(state); + + if (result.action === 'need-more-data') { + const got = await pullChunk(); + if (!got) { + throw new Error('Unexpected end of archive'); + } + continue; + } + + if (result.action === 'end-of-archive') { + yield { kind: 'end' }; + return; + } + + if (result.action === 'pax-consumed') { + continue; + } + + // action === 'entry' + const entry = result.entry; + yield { kind: 'entry', entry }; + + if (mode === 'list' || entry.size === 0) { + // For list mode or empty entries, skip content + padding together. + if (entry.size > 0) { + bytesRemaining = entry.size + calculatePadding(entry.size); + phase = 'SKIP'; + } else { + phase = 'HEADER'; + } + } else { + // Extract mode with content. + bytesRemaining = entry.size; + paddingRemaining = calculatePadding(entry.size); + phase = 'CONTENT'; + } + break; + } + + continue; + } + + if (phase === 'CONTENT') { + // Yield content bytes, splitting at entry boundary. + if (bytesRemaining === 0) { + phase = 'PADDING'; + continue; + } + + if (state.buffer.length === 0) { + const got = await pullChunk(); + if (!got) { + throw new Error('Unexpected end of archive'); + } + } + + const take = Math.min(bytesRemaining, state.buffer.length); + const chunk = state.buffer.subarray(0, take); + state.buffer = state.buffer.subarray(take); + bytesRemaining -= take; + yield { + kind: 'chunk', + data: new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength), + }; + continue; + } + + if (phase === 'SKIP') { + // Silently consume skip bytes (list mode or zero-size entries). + if (bytesRemaining === 0) { + phase = 'HEADER'; + continue; + } + + if (state.buffer.length === 0) { + const got = await pullChunk(); + if (!got) { + throw new Error('Unexpected end of archive'); + } + } + + const skip = Math.min(bytesRemaining, state.buffer.length); + state.buffer = state.buffer.subarray(skip); + bytesRemaining -= skip; + continue; + } + + if (phase === 'PADDING') { + // Silently consume padding bytes. + if (paddingRemaining === 0) { + phase = 'HEADER'; + continue; + } + + if (state.buffer.length === 0) { + const got = await pullChunk(); + if (!got) { + // Padding missing at end-of-stream is tolerable if we already know + // the archive ended (previous end-of-archive detection covers the + // normal case). If we're here the EOA hasn't been seen yet — throw. + throw new Error('Unexpected end of archive'); + } + } + + const skip = Math.min(paddingRemaining, state.buffer.length); + state.buffer = state.buffer.subarray(skip); + paddingRemaining -= skip; + continue; + } + + // Unreachable — all phases handled above. + /* v8 ignore next */ + break; + } + } finally { + // If the consumer called generator.return() early, propagate cleanup to the + // source iterator so upstream (streamXz) can destroy its pipeline. + await iter.return?.(); + } +} diff --git a/packages/tar-xz/src/node/xz-helpers.ts b/packages/tar-xz/src/node/xz-helpers.ts index 515f658..28254c3 100644 --- a/packages/tar-xz/src/node/xz-helpers.ts +++ b/packages/tar-xz/src/node/xz-helpers.ts @@ -1,74 +1,71 @@ /** * Shared Node.js helpers for XZ decompression pipelines. * - * Used by both `list.ts` and `extract.ts` to avoid duplication. + * `streamXz` is the primary entry point — used by `extract.ts` and `list.ts`. * These helpers depend on `node:stream` and are not suitable for browser bundles. */ -import { Readable, type Writable } from 'node:stream'; +import { Readable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; import { createUnxz } from 'node-liblzma'; import { toAsyncIterable, type TarInputNode } from '../internal/to-async-iterable.js'; -/** Collect all chunks from any TarInputNode into a single Uint8Array. */ -export async function collectAllChunks(input: TarInputNode): Promise { - const iterable = toAsyncIterable(input); - const chunks: Uint8Array[] = []; - for await (const chunk of iterable) { - chunks.push(chunk); - } - const total = chunks.reduce((n, c) => n + c.length, 0); - const out = new Uint8Array(total); - let offset = 0; - for (const chunk of chunks) { - out.set(chunk, offset); - offset += chunk.length; - } - return out; -} - -/** Decompress XZ data using the Node Transform stream. */ -export async function decompressXz(data: Uint8Array): Promise { - const unxzStream = createUnxz(); - const readable = Readable.from( - (async function* () { - yield data; - })() - ); - - const output: Uint8Array[] = []; - let resolveFlush!: () => void; - let rejectFlush!: (e: unknown) => void; - const done = new Promise((res, rej) => { - resolveFlush = res; - rejectFlush = rej; - }); - - unxzStream.on('data', (...args: unknown[]) => { - const chunk = args[0] as Buffer; - output.push(new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength)); - }); - unxzStream.on('end', resolveFlush); - unxzStream.on('error', rejectFlush); - readable.pipe(unxzStream); +/** + * Decompress an XZ-compressed input stream incrementally, yielding raw tar bytes + * as they arrive from the decompressor. No full-archive accumulation. + * + * Accepts any {@link TarInputNode}: `Uint8Array`, `Buffer`, `Readable`, + * `AsyncIterable`, `ReadableStream`, or `ArrayBuffer`. + * + * The returned `AsyncIterable` is single-use. Errors (corrupt XZ, truncated + * input) are thrown inside the `for await` loop. + * + * @example + * ```ts + * for await (const chunk of streamXz(input)) { + * processChunk(chunk); + * } + * ``` + */ +export function streamXz(input: TarInputNode): AsyncIterable { + // Convert any supported input type to an AsyncIterable. + const source = toAsyncIterable(input); - await done; + // Pipeline and Transform are created INSIDE the generator body so that no + // I/O starts until the consumer calls .next() for the first time (true lazy + // semantics). If the caller never iterates, neither the pipeline nor the + // Transform stream are ever created — no unhandled rejections, no resource leak. + return (async function* () { + const unxzStream = createUnxz(); - const total = output.reduce((n, c) => n + c.length, 0); - const result = new Uint8Array(total); - let offset = 0; - for (const chunk of output) { - result.set(chunk, offset); - offset += chunk.length; - } - return result; -} + // Feed the source into the Transform via pipeline() so that errors from the + // input side (e.g. truncated readable) propagate and are not silently swallowed. + // We do NOT await pipelinePromise here — iteration drives consumption below. + const pipelinePromise = pipeline(Readable.from(source), unxzStream); -/** Feed a Uint8Array into a Writable and wait for finish. */ -export async function runWritable(writable: Writable, data: Uint8Array): Promise { - await new Promise((resolve, reject) => { - writable.on('finish', resolve); - writable.on('error', reject); - writable.write(Buffer.from(data.buffer, data.byteOffset, data.byteLength)); - writable.end(); - }); + // Iterate the Transform's own Symbol.asyncIterator directly (spec §12.5: + // prefer this over Readable.from(transform) which adds another buffering layer). + // Node's Transform implements Symbol.asyncIterator natively since Node 10. + try { + for await (const chunk of unxzStream) { + const buf = chunk as Buffer; + yield new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength); + } + // Surface any pipeline error (e.g. corrupt XZ, premature end of source). + await pipelinePromise; + } catch (err) { + // Ensure the pipeline promise is always settled to avoid unhandled rejections. + await pipelinePromise.catch(() => undefined); + throw err; + } finally { + // If the consumer stopped iterating early (generator.return() was called), + // destroy the Transform stream so the pipeline does not keep running. + // destroy() is idempotent — safe to call even if the stream already ended. + if (!unxzStream.destroyed) { + unxzStream.destroy(); + } + // Swallow any subsequent pipeline rejection caused by the destroy() above. + await pipelinePromise.catch(() => undefined); + } + })(); } diff --git a/packages/tar-xz/test/coverage.spec.ts b/packages/tar-xz/test/coverage.spec.ts index 4cb75f8..05a5fbb 100644 --- a/packages/tar-xz/test/coverage.spec.ts +++ b/packages/tar-xz/test/coverage.spec.ts @@ -7,7 +7,7 @@ import * as os from 'node:os'; import * as path from 'node:path'; import { xzSync } from 'node-liblzma'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { extract } from '../src/node/index.js'; +import { extract, list } from '../src/node/index.js'; import { create } from '../src/node/create.js'; import { createFile, extractFile, listFile } from '../src/node/file.js'; import { parseOctal } from '../src/tar/checksum.js'; @@ -1163,3 +1163,144 @@ describe('Coverage: Node API', () => { }); }); }); + +// --------------------------------------------------------------------------- +// Block 3 — extract() streaming-specific tests (S-08, D-3, S-08b) +// --------------------------------------------------------------------------- + +describe('extract() streaming behaviours (Block 3)', () => { + let tempDir: string; + + beforeEach(async () => { + tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'tar-xz-b3-test-')); + }); + + afterEach(async () => { + await fs.rm(tempDir, { recursive: true, force: true }); + }); + + async function buildAndSaveArchive( + files: Array<{ name: string; content: Buffer }> + ): Promise { + const archive = path.join(tempDir, 'archive.tar.xz'); + const chunks: Uint8Array[] = []; + for await (const chunk of create({ + files: files.map((f) => ({ name: f.name, source: f.content })), + })) { + chunks.push(chunk); + } + await fs.writeFile(archive, Buffer.concat(chunks.map((c) => Buffer.from(c)))); + return archive; + } + + it('S-08: consumer skips entry.data — next entry yields correct content', async () => { + const archive = await buildAndSaveArchive([ + { name: 'skip-me.bin', content: Buffer.alloc(102400, 0xaa) }, // 100 KB + { name: 'read-me.txt', content: Buffer.from('hello-streaming') }, + ]); + + const entries: Array<{ name: string; content: string }> = []; + for await (const entry of extract(createReadStream(archive))) { + if (entry.name === 'skip-me.bin') { + // Deliberately do NOT call entry.bytes() or iterate entry.data. + continue; // auto-drain kicks in + } + entries.push({ name: entry.name, content: await entry.text() }); + } + + expect(entries).toHaveLength(1); + expect(entries[0]?.name).toBe('read-me.txt'); + expect(entries[0]?.content).toBe('hello-streaming'); + }); + + it('D-3: bytes() memoization — second call returns same buffer instance', async () => { + const archive = await buildAndSaveArchive([ + { name: 'file.txt', content: Buffer.from('memoize-me') }, + ]); + + for await (const entry of extract(createReadStream(archive))) { + const first = await entry.bytes(); + const second = await entry.bytes(); + expect(first).toBe(second); // same reference — memoized + expect(Buffer.from(first).toString()).toBe('memoize-me'); + } + }); + + it('D-3: data iterated AFTER bytes() yields nothing', async () => { + const archive = await buildAndSaveArchive([ + { name: 'file.txt', content: Buffer.from('once-only') }, + ]); + + for await (const entry of extract(createReadStream(archive))) { + await entry.bytes(); // consumes the data generator + const chunks: Uint8Array[] = []; + for await (const chunk of entry.data) { + chunks.push(chunk); + } + // Generator is exhausted; second iteration yields nothing. + expect(chunks).toHaveLength(0); + } + }); + + it('S-08b consumer-break (D-2): for-await break does NOT surface errors from skipped data', async () => { + const archive = await buildAndSaveArchive([ + { name: 'a.txt', content: Buffer.from('first') }, + { name: 'b.txt', content: Buffer.from('second') }, + { name: 'c.txt', content: Buffer.from('third') }, + ]); + + const names: string[] = []; + // Break after first entry — should not throw. + for await (const entry of extract(createReadStream(archive))) { + names.push(entry.name); + break; // consumer-break — triggers generator finally + } + + expect(names).toHaveLength(1); + expect(names[0]).toBe('a.txt'); + }); +}); + +// --------------------------------------------------------------------------- +// Block 4 — list() streaming-specific tests (S-12 placeholder) +// --------------------------------------------------------------------------- + +describe('list() streaming behaviours (Block 4)', () => { + let tempDir: string; + + beforeEach(async () => { + tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'tar-xz-b4-test-')); + }); + + afterEach(async () => { + await fs.rm(tempDir, { recursive: true, force: true }); + }); + + it('S-12 placeholder: list() yields all entries with correct metadata', async () => { + const archive = path.join(tempDir, 'archive.tar.xz'); + const chunks: Uint8Array[] = []; + for await (const chunk of create({ + files: [ + { name: 'alpha.txt', source: Buffer.from('alpha') }, + { name: 'beta.txt', source: Buffer.from('beta-content-here') }, + { name: 'gamma.txt', source: Buffer.from('g') }, + ], + })) { + chunks.push(chunk); + } + await fs.writeFile(archive, Buffer.concat(chunks.map((c) => Buffer.from(c)))); + + const entries: TarEntry[] = []; + for await (const entry of list(createReadStream(archive))) { + entries.push(entry); + } + + expect(entries).toHaveLength(3); + expect(entries[0]?.name).toBe('alpha.txt'); + expect(entries[0]?.size).toBe(5); + expect(entries[1]?.name).toBe('beta.txt'); + expect(entries[1]?.size).toBe(17); + expect(entries[2]?.name).toBe('gamma.txt'); + expect(entries[2]?.size).toBe(1); + }); +}); diff --git a/packages/tar-xz/test/memory-shape.spec.ts b/packages/tar-xz/test/memory-shape.spec.ts new file mode 100644 index 0000000..52f2ab8 --- /dev/null +++ b/packages/tar-xz/test/memory-shape.spec.ts @@ -0,0 +1,240 @@ +/** + * Memory shape CI gate — Block 5 (TAR-XZ-STREAMING-2026-04-28) + * + * Validates that the streaming extract() / list() implementation is O(largest entry) + * rather than O(archive), using in-loop high-water-mark sampling (§12.3). + * + * Test runner requirements: + * - vitest pool: 'forks' with execArgv: ['--expose-gc'] (vitest.config.ts) + * - These tests feature-detect `globalThis.gc` at runtime; they SKIP (not fail) + * when --expose-gc is absent, so CI remains green without the flag. + * + * Thresholds (per §12.3): + * - extract: peak delta ≤ 2 × largestEntrySize + 16 MB + * (16 MB slack covers XZ preset-6 ~8 MB dictionary + parser carry-over + GC jitter) + * - list: peak delta ≤ 16 MB regardless of total archive size + * (list never holds entry data — O(BLOCK_SIZE) memory) + */ + +import { Readable } from 'node:stream'; +import { describe, expect, it } from 'vitest'; +import { create, extract, list } from '../src/node/index.js'; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/** High-water mark sampler. Call on every chunk boundary inside the test loop. */ +function makeSampler(): { sample: () => void; peak: () => number; baseline: number } { + const m = process.memoryUsage(); + const baseline = m.heapUsed + m.external; + let high = baseline; + + return { + baseline, + sample() { + const mu = process.memoryUsage(); + const total = mu.heapUsed + mu.external; + if (total > high) high = total; + }, + peak() { + return high; + }, + }; +} + +/** + * Build an in-memory .tar.xz archive from a list of { name, size } entries. + * Each file is filled with a repeated byte pattern (compressible). + * Returns a Uint8Array of the compressed archive. + */ +async function buildArchive( + files: Array<{ name: string; size: number; byte?: number }>, + preset = 1 +): Promise { + const chunks: Uint8Array[] = []; + for await (const chunk of create({ + files: files.map((f) => ({ + name: f.name, + source: Buffer.alloc(f.size, f.byte ?? 0xab), + })), + preset, // Default fast preset; callers may pass 6 when testing preset-6 dictionary slack + })) { + chunks.push(chunk); + } + const total = chunks.reduce((n, c) => n + c.length, 0); + const out = new Uint8Array(total); + let offset = 0; + for (const c of chunks) { + out.set(c, offset); + offset += c.length; + } + return out; +} + +// Feature-detect --expose-gc (provided by vitest forks pool config) +const hasGc = typeof (globalThis as unknown as { gc?: () => void }).gc === 'function'; +const gc = hasGc ? (globalThis as unknown as { gc: () => void }).gc : null; + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +describe('Memory shape gate (requires --expose-gc; skips otherwise)', () => { + // ------------------------------------------------------------------------- + // Test 1: extract() with a single 50 MB entry + // ------------------------------------------------------------------------- + + it.runIf(hasGc)( + 'extract memory shape — single 50 MB entry — peak delta ≤ 2× entry + 16 MB slack', + async () => { + const ENTRY_SIZE = 50 * 1024 * 1024; // 50 MB + const THRESHOLD = 2 * ENTRY_SIZE + 16 * 1024 * 1024; // 116 MB + + // preset: 6 matches the 16 MB slack rationale (XZ preset-6 dictionary ≈ 8 MB) + const archive = await buildArchive([{ name: 'big.bin', size: ENTRY_SIZE }], 6); + + gc!(); + const sampler = makeSampler(); + + for await (const entry of extract(Readable.from([archive]))) { + sampler.sample(); + for await (const _ of entry.data) { + sampler.sample(); + } + sampler.sample(); + } + + gc!(); + sampler.sample(); + + const delta = sampler.peak() - sampler.baseline; + + // eslint-disable-next-line no-console + console.log( + `[memory-shape] extract single-entry: delta=${(delta / 1024 / 1024).toFixed(1)} MB, ` + + `threshold=${(THRESHOLD / 1024 / 1024).toFixed(0)} MB` + ); + + expect(delta).toBeLessThan(THRESHOLD); + }, + // 60-second timeout for large archive build + extract + 60_000 + ); + + // ------------------------------------------------------------------------- + // Test 2: list() with 100 entries — peak bounded regardless of total size + // ------------------------------------------------------------------------- + + it.runIf(hasGc)( + 'list memory shape — 100 × 1 MB entries — peak delta ≤ 16 MB', + async () => { + // 100 entries × 1 MB = 100 MB total decompressed size + // list() should hold O(BLOCK_SIZE) — no content bytes, only header metadata + const ENTRY_COUNT = 100; + const ENTRY_SIZE = 1 * 1024 * 1024; // 1 MB each + const THRESHOLD = 16 * 1024 * 1024; // 16 MB + + const archive = await buildArchive( + Array.from({ length: ENTRY_COUNT }, (_, i) => ({ + name: `file-${i.toString().padStart(3, '0')}.bin`, + size: ENTRY_SIZE, + })) + ); + + gc!(); + const sampler = makeSampler(); + + let count = 0; + for await (const entry of list(Readable.from([archive]))) { + sampler.sample(); + count++; + expect(typeof entry.name).toBe('string'); + expect(entry.size).toBe(ENTRY_SIZE); + } + + gc!(); + sampler.sample(); + + expect(count).toBe(ENTRY_COUNT); + + const delta = sampler.peak() - sampler.baseline; + + // eslint-disable-next-line no-console + console.log( + `[memory-shape] list 100×1MB: delta=${(delta / 1024 / 1024).toFixed(1)} MB, ` + + `threshold=${(THRESHOLD / 1024 / 1024).toFixed(0)} MB` + ); + + expect(delta).toBeLessThan(THRESHOLD); + }, + 90_000 + ); + + // ------------------------------------------------------------------------- + // Test 3: extract() with 5 × 10 MB entries — peak bounded to largest entry + // ------------------------------------------------------------------------- + + it.runIf(hasGc)( + 'extract memory shape — 5 × 10 MB entries — peak delta ≤ 2× largest + 16 MB slack', + async () => { + const ENTRY_COUNT = 5; + const LARGEST_ENTRY = 10 * 1024 * 1024; // 10 MB + const THRESHOLD = 2 * LARGEST_ENTRY + 16 * 1024 * 1024; // 36 MB + + const archive = await buildArchive( + Array.from({ length: ENTRY_COUNT }, (_, i) => ({ + name: `chunk-${i}.bin`, + size: LARGEST_ENTRY, + byte: i * 17, // different byte per entry so XZ can't trivially dedupe + })) + ); + + gc!(); + const sampler = makeSampler(); + + let totalBytes = 0; + for await (const entry of extract(Readable.from([archive]))) { + sampler.sample(); + for await (const chunk of entry.data) { + sampler.sample(); + totalBytes += chunk.length; + } + sampler.sample(); + } + + gc!(); + sampler.sample(); + + expect(totalBytes).toBe(ENTRY_COUNT * LARGEST_ENTRY); + + const delta = sampler.peak() - sampler.baseline; + + // eslint-disable-next-line no-console + console.log( + `[memory-shape] extract 5×10MB: delta=${(delta / 1024 / 1024).toFixed(1)} MB, ` + + `threshold=${(THRESHOLD / 1024 / 1024).toFixed(0)} MB` + ); + + expect(delta).toBeLessThan(THRESHOLD); + }, + 90_000 + ); + + // ------------------------------------------------------------------------- + // Informational: skip message when --expose-gc is not available + // ------------------------------------------------------------------------- + + it.runIf(!hasGc)( + 'memory shape tests SKIPPED — run with --expose-gc (vitest forks pool: execArgv)', + () => { + console.log( + '[memory-shape] SKIPPED: globalThis.gc is not a function. ' + + 'Ensure vitest.config.ts pool="forks" with execArgv=["--expose-gc"] ' + + 'and run via: pnpm test:memory' + ); + // This test passes trivially — it exists only to emit the skip message. + expect(hasGc).toBe(false); + } + ); +}); diff --git a/packages/tar-xz/test/node-api.spec.ts b/packages/tar-xz/test/node-api.spec.ts index 5854430..062a6af 100644 --- a/packages/tar-xz/test/node-api.spec.ts +++ b/packages/tar-xz/test/node-api.spec.ts @@ -235,6 +235,119 @@ describe('Node.js file API (v6)', () => { }); }); + describe('entry helpers (bytes / text)', () => { + it('F-2: bytes() throws when entry.data was already iterated', async () => { + const { create } = await import('../src/node/create.js'); + const { extract } = await import('../src/node/extract.js'); + + const archive = create({ + files: [{ name: 'test.txt', source: Buffer.from('hello') }], + }); + + for await (const entry of extract(archive)) { + // Partially consume entry.data first + // biome-ignore lint/suspicious/noEmptyBlockStatements: intentional drain + for await (const _ of entry.data) { + /* drain */ + } + // Now bytes() must throw since dataGen is consumed + await expect(entry.bytes()).rejects.toThrow(/entry\.data already iterated/); + } + }); + + it('F-2: bytes() works fine when entry.data was NOT iterated', async () => { + const { create } = await import('../src/node/create.js'); + const { extract } = await import('../src/node/extract.js'); + + const content = Buffer.from('hello world'); + const archive = create({ + files: [{ name: 'test.txt', source: content }], + }); + + for await (const entry of extract(archive)) { + const result = await entry.bytes(); + expect(Buffer.from(result)).toEqual(content); + } + }); + + it('F-3: text() with utf8 encoding decodes correctly', async () => { + const { create } = await import('../src/node/create.js'); + const { extract } = await import('../src/node/extract.js'); + + const content = 'Hello, World!'; + const archive = create({ + files: [{ name: 'test.txt', source: Buffer.from(content, 'utf8') }], + }); + + for await (const entry of extract(archive)) { + expect(await entry.text()).toBe(content); + expect(await entry.text('utf8')).toBe(content); + } + }); + + it('F-3: text("base64") base64-encodes the content (Buffer.toString contract)', async () => { + const { create } = await import('../src/node/create.js'); + const { extract } = await import('../src/node/extract.js'); + + const content = Buffer.from('binary data'); + const expected = content.toString('base64'); + + const archive = create({ + files: [{ name: 'data.bin', source: content }], + }); + + for await (const entry of extract(archive)) { + expect(await entry.text('base64')).toBe(expected); + } + }); + + it('F-3: text("hex") hex-encodes the content (Buffer.toString contract)', async () => { + const { create } = await import('../src/node/create.js'); + const { extract } = await import('../src/node/extract.js'); + + const content = Buffer.from([0xde, 0xad, 0xbe, 0xef]); + const expected = content.toString('hex'); + + const archive = create({ + files: [{ name: 'bytes.bin', source: content }], + }); + + for await (const entry of extract(archive)) { + expect(await entry.text('hex')).toBe(expected); + } + }); + + it('F-1: breaking out of extract() early causes no unhandled rejection', async () => { + const { create } = await import('../src/node/create.js'); + const { extract } = await import('../src/node/extract.js'); + + const archive = create({ + files: [ + { name: 'a.txt', source: Buffer.from('entry-a') }, + { name: 'b.txt', source: Buffer.from('entry-b') }, + { name: 'c.txt', source: Buffer.from('entry-c') }, + ], + }); + + const unhandledRejections: Error[] = []; + const handler = (err: Error) => unhandledRejections.push(err); + process.on('unhandledRejection', handler); + + try { + for await (const entry of extract(archive)) { + expect(entry.name).toBe('a.txt'); + break; // consumer breaks after first entry + } + // Allow a microtask turn for any unhandled rejection to surface + await new Promise((r) => setTimeout(r, 20)); + } finally { + process.off('unhandledRejection', handler); + } + + expect(unhandledRejections).toHaveLength(0); + }); + }); + describe('streaming (in-memory)', () => { it('create() yields compressed chunks without disk I/O', async () => { const { create } = await import('../src/node/create.js'); diff --git a/packages/tar-xz/test/security.spec.ts b/packages/tar-xz/test/security.spec.ts new file mode 100644 index 0000000..eed552c --- /dev/null +++ b/packages/tar-xz/test/security.spec.ts @@ -0,0 +1,723 @@ +/** + * Security regression gate — Block 5 (TAR-XZ-STREAMING-2026-04-28) + * + * Consolidates all 18 TOCTOU + safety vectors into a single labelled file so + * that any future change to extract.ts / list.ts / file.ts / xz-helpers.ts + * that breaks a security invariant fails here with an unambiguous test name. + * + * Vector map: + * V1/R6-1 — leaf symlink check → ensureSafeTarget + * F-001 — traversal rel === '..' → ensureSafeTarget + * F-002 — TOCTOU ancestor check → hasSymlinkAncestor + * R4-2 — hardlink strip logic → extractFile + * R5-1 — hardlink symlink source → extractFile + * V6a/V6b — NUL/empty name → ensureSafeName + * V12 — setuid mask → extractFile + * S2 — hardlink escape → extractFile + * S3 — symlink ancestor TOCTOU → hasSymlinkAncestor + * V2/V3 — fd-based O_NOFOLLOW → extractFile + * S-14 — Windows TOCTOU window policy → threat-model doc note + * S-15 — PAX bomb DoS → parseTar MAX_PAX_HEADER_BYTES + */ + +import { createReadStream } from 'node:fs'; +import { promises as fs } from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { xzSync } from 'node-liblzma'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { extractFile } from '../src/node/file.js'; +import { extract } from '../src/node/index.js'; +import { + BLOCK_SIZE, + calculatePadding, + createEndOfArchive, + createHeader, +} from '../src/tar/format.js'; +import { createPaxHeaderBlocks } from '../src/tar/pax.js'; +import { TarEntryType } from '../src/types.js'; + +// --------------------------------------------------------------------------- +// Local test helpers (mirrors coverage.spec.ts helpers — intentionally local +// so this file is self-contained as a regression lock). +// --------------------------------------------------------------------------- + +/** Build a raw TAR buffer from entry descriptors. */ +function buildTar( + entries: Array<{ + name: string; + content?: Buffer; + type?: string; + linkname?: string; + usePax?: boolean; + mode?: number; + }> +): Buffer { + const blocks: Buffer[] = []; + + for (const entry of entries) { + const content = entry.content ?? Buffer.alloc(0); + const type = (entry.type ?? TarEntryType.FILE) as string; + const isDir = type === TarEntryType.DIRECTORY; + const isLink = type === TarEntryType.SYMLINK || type === TarEntryType.HARDLINK; + const size = isDir || isLink ? 0 : content.length; + + let headerName = entry.name; + + if (entry.usePax || headerName.length > 255) { + const paxBlocks = createPaxHeaderBlocks(headerName, { + path: headerName, + size, + linkpath: entry.linkname, + }); + for (const block of paxBlocks) { + blocks.push(Buffer.from(block)); + } + headerName = headerName.slice(-99); + } + + const header = createHeader({ + name: headerName, + size, + type: type as '0', + linkname: entry.linkname, + mode: entry.mode, + }); + blocks.push(Buffer.from(header)); + + if (size > 0) { + blocks.push(content); + const pad = calculatePadding(size); + if (pad > 0) blocks.push(Buffer.alloc(pad)); + } + } + + blocks.push(Buffer.from(createEndOfArchive())); + return Buffer.concat(blocks); +} + +/** Compress a raw TAR buffer to .tar.xz and write to archivePath. */ +async function saveAsXz(tar: Buffer | Uint8Array, archivePath: string): Promise { + await fs.writeFile(archivePath, xzSync(Buffer.isBuffer(tar) ? tar : Buffer.from(tar))); +} + +/** Build a .tar.xz archive containing a crafted PAX header claiming a giant payload. + * + * The returned Buffer is a valid XZ stream containing a TAR archive where the + * PAX_HEADER entry declares `size = claimedSize` bytes but only `actualBytes` + * bytes of PAX data are provided. + */ +function buildPaxBombTar(claimedSize: number, actualBytes: number): Buffer { + const blocks: Buffer[] = []; + + // PAX extended header entry claiming `claimedSize` bytes + const paxHeader = createHeader({ + name: 'PaxHeaders/bomb.txt', + type: TarEntryType.PAX_HEADER, + size: claimedSize, + }); + blocks.push(Buffer.from(paxHeader)); + + // Only actualBytes of payload (truncated PAX data) + const fakePayload = Buffer.alloc(actualBytes, 0x20); // spaces + blocks.push(fakePayload); + if (actualBytes > 0) { + const pad = calculatePadding(actualBytes); + if (pad > 0) blocks.push(Buffer.alloc(pad)); + } + + blocks.push(Buffer.from(createEndOfArchive())); + return Buffer.concat(blocks); +} + +// --------------------------------------------------------------------------- +// Test suite +// --------------------------------------------------------------------------- + +describe('Security regression gate', () => { + let tempDir: string; + + beforeEach(async () => { + tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'tar-xz-sec-')); + }); + + afterEach(async () => { + await fs.rm(tempDir, { recursive: true, force: true }).catch(() => {}); + }); + + // ------------------------------------------------------------------------- + // V1/R6-1 — Leaf symlink check (ensureSafeTarget) + // ------------------------------------------------------------------------- + + describe('V1/R6-1: leaf symlink check (ensureSafeTarget)', () => { + it('V1/R6-1 FILE: rejects overwrite when target is a pre-existing symlink', async () => { + const dest = path.join(tempDir, 'dest'); + await fs.mkdir(dest); + + const externalDir = path.join(tempDir, 'external'); + await fs.mkdir(externalDir); + const secretFile = path.join(externalDir, 'secret'); + await fs.writeFile(secretFile, 'original-secret'); + + // archive: SYMLINK evil → ../external/secret, then FILE evil (attempts overwrite) + const tar = buildTar([ + { name: 'evil', type: TarEntryType.SYMLINK, linkname: '../external/secret' }, + { name: 'evil', content: Buffer.from('overwritten') }, + ]); + const archive = path.join(tempDir, 'leaf-file.tar.xz'); + await saveAsXz(tar, archive); + + await expect(extractFile(archive, { cwd: dest })).rejects.toThrow(/symlink/i); + + // External file must NOT be overwritten + expect(await fs.readFile(secretFile, 'utf8')).toBe('original-secret'); + }); + + it('V1/R6-1 DIRECTORY: rejects when target is a pre-existing symlink', async () => { + const dest = path.join(tempDir, 'dest'); + await fs.mkdir(dest); + + const externalDir = path.join(tempDir, 'external'); + await fs.mkdir(externalDir); + + const tar = buildTar([ + { name: 'evil', type: TarEntryType.SYMLINK, linkname: '../external' }, + { name: 'evil/', type: TarEntryType.DIRECTORY }, + ]); + const archive = path.join(tempDir, 'leaf-dir.tar.xz'); + await saveAsXz(tar, archive); + + await expect(extractFile(archive, { cwd: dest })).rejects.toThrow(/symlink/i); + }); + }); + + // ------------------------------------------------------------------------- + // F-001 — Path traversal rel === '..' (ensureSafeTarget) + // ------------------------------------------------------------------------- + + describe('F-001: path traversal via ".." segments (ensureSafeTarget)', () => { + it('F-001: rejects ../../../escape style path traversal', async () => { + const tar = buildTar([{ name: '../../../tmp/evil.txt', content: Buffer.from('evil') }]); + const archive = path.join(tempDir, 'traversal.tar.xz'); + await saveAsXz(tar, archive); + + const dest = path.join(tempDir, 'dest'); + await fs.mkdir(dest); + + await expect(extractFile(archive, { cwd: dest })).rejects.toThrow(/outside cwd/i); + }); + + it('F-001: rejects entry named ".." (dot-dot directory)', async () => { + const tar = buildTar([{ name: '..', type: TarEntryType.DIRECTORY }]); + const archive = path.join(tempDir, 'dotdot.tar.xz'); + await saveAsXz(tar, archive); + + const dest = path.join(tempDir, 'dest'); + await fs.mkdir(dest); + + // Either "outside cwd" or "dot-segment" rejection is acceptable + await expect(extractFile(archive, { cwd: dest })).rejects.toThrow(); + }); + + it('F-001: does NOT reject legitimate dotfiles like ".gitignore"', async () => { + const tar = buildTar([{ name: '.gitignore', content: Buffer.from('*.log') }]); + const archive = path.join(tempDir, 'dotfile.tar.xz'); + await saveAsXz(tar, archive); + + const dest = path.join(tempDir, 'dest'); + await fs.mkdir(dest); + + await extractFile(archive, { cwd: dest }); + expect(await fs.readFile(path.join(dest, '.gitignore'), 'utf8')).toBe('*.log'); + }); + }); + + // ------------------------------------------------------------------------- + // F-002 — TOCTOU ancestor check (hasSymlinkAncestor) + // ------------------------------------------------------------------------- + + describe('F-002: TOCTOU ancestor check (hasSymlinkAncestor)', () => { + it('F-002: rejects file written through symlink ancestor (S3 vector)', async () => { + const dest = path.join(tempDir, 'dest'); + await fs.mkdir(dest); + + const externalDir = path.join(tempDir, 'external'); + await fs.mkdir(externalDir); + + // symlink 'link' → '../external', then 'link/file.txt' through it + const tar = buildTar([ + { name: 'link', type: TarEntryType.SYMLINK, linkname: '../external' }, + { name: 'link/file.txt', content: Buffer.from('escaped') }, + ]); + const archive = path.join(tempDir, 'toctou.tar.xz'); + await saveAsXz(tar, archive); + + await expect(extractFile(archive, { cwd: dest })).rejects.toThrow(/symlink/i); + + // External directory must remain empty + expect(await fs.readdir(externalDir)).toHaveLength(0); + }); + + it('F-002: rejects directory created through symlink ancestor', async () => { + const dest = path.join(tempDir, 'dest'); + await fs.mkdir(dest); + + const externalDir = path.join(tempDir, 'external'); + await fs.mkdir(externalDir); + + const tar = buildTar([ + { name: 'link', type: TarEntryType.SYMLINK, linkname: '../external' }, + { name: 'link/subdir/', type: TarEntryType.DIRECTORY }, + ]); + const archive = path.join(tempDir, 'dir-toctou.tar.xz'); + await saveAsXz(tar, archive); + + await expect(extractFile(archive, { cwd: dest })).rejects.toThrow(/symlink/i); + + // external/subdir must NOT have been created + expect(await fs.readdir(externalDir)).toHaveLength(0); + }); + + it('F-002 / R3-1: walks past ENOENT when checking ancestor symlinks', async () => { + // Regression: old code returned false on ENOENT in hasSymlinkAncestor, + // stopping the walk early. Archive: link→../external, then link/subdir/file.txt + // (link/subdir does NOT exist). Walk must continue past the ENOENT. + const dest = path.join(tempDir, 'dest'); + await fs.mkdir(dest); + + const externalDir = path.join(tempDir, 'external'); + await fs.mkdir(externalDir); + + const tar = buildTar([ + { name: 'link', type: TarEntryType.SYMLINK, linkname: '../external' }, + { name: 'link/subdir/file.txt', content: Buffer.from('escaped') }, + ]); + const archive = path.join(tempDir, 'enoent-toctou.tar.xz'); + await saveAsXz(tar, archive); + + await expect(extractFile(archive, { cwd: dest })).rejects.toThrow(/symlink/i); + + // External must be empty + expect(await fs.readdir(externalDir)).toHaveLength(0); + }); + }); + + // ------------------------------------------------------------------------- + // S3 — Symlink ancestor TOCTOU (hasSymlinkAncestor) — comprehensive variants + // ------------------------------------------------------------------------- + + describe('S3: symlink ancestor TOCTOU — all entry types', () => { + it('S3 SYMLINK: rejects symlink entry whose path has a symlink ancestor', async () => { + const dest = path.join(tempDir, 'dest'); + await fs.mkdir(dest); + + const externalDir = path.join(tempDir, 'external'); + await fs.mkdir(externalDir); + + const tar = buildTar([ + { name: 'link', type: TarEntryType.SYMLINK, linkname: '../external' }, + { name: 'link/inner', type: TarEntryType.SYMLINK, linkname: 'anything' }, + ]); + const archive = path.join(tempDir, 'sym-toctou.tar.xz'); + await saveAsXz(tar, archive); + + await expect(extractFile(archive, { cwd: dest })).rejects.toThrow(/symlink/i); + expect(await fs.readdir(externalDir)).toHaveLength(0); + }); + + it('S3 HARDLINK: rejects hardlink entry whose target path has a symlink ancestor', async () => { + const dest = path.join(tempDir, 'dest'); + await fs.mkdir(dest); + + const externalDir = path.join(tempDir, 'external'); + await fs.mkdir(externalDir); + + const tar = buildTar([ + { name: 'original.txt', content: Buffer.from('data') }, + { name: 'link', type: TarEntryType.SYMLINK, linkname: '../external' }, + { name: 'link/file.txt', type: TarEntryType.HARDLINK, linkname: 'original.txt' }, + ]); + const archive = path.join(tempDir, 'hl-toctou.tar.xz'); + await saveAsXz(tar, archive); + + await expect(extractFile(archive, { cwd: dest })).rejects.toThrow(/symlink/i); + expect(await fs.readdir(externalDir)).toHaveLength(0); + }); + }); + + // ------------------------------------------------------------------------- + // R4-2 — Hardlink strip logic (extractFile) + // ------------------------------------------------------------------------- + + describe('R4-2: hardlink strip logic (extractFile)', () => { + it('R4-2: strip option applies to hardlink linkname (not just entry name)', async () => { + const dest = path.join(tempDir, 'dest'); + await fs.mkdir(dest); + + const tar = buildTar([ + { name: 'dir/a.txt', content: Buffer.from('content') }, + { name: 'dir/link', type: TarEntryType.HARDLINK, linkname: 'dir/a.txt' }, + ]); + const archive = path.join(tempDir, 'hl-strip.tar.xz'); + await saveAsXz(tar, archive); + + // strip: 1 → 'dir/a.txt' → 'a.txt'; 'dir/link' → 'link'; linkname 'dir/a.txt' → 'a.txt' + await extractFile(archive, { cwd: dest, strip: 1 }); + + const aPath = path.join(dest, 'a.txt'); + const linkPath = path.join(dest, 'link'); + expect(await fs.readFile(aPath, 'utf8')).toBe('content'); + expect(await fs.readFile(linkPath, 'utf8')).toBe('content'); + + // Confirm hardlink semantics: same inode + const aStat = await fs.stat(aPath); + const linkStat = await fs.stat(linkPath); + expect(linkStat.ino).toBe(aStat.ino); + }); + }); + + // ------------------------------------------------------------------------- + // R5-1 — Hardlink symlink source (extractFile) + // ------------------------------------------------------------------------- + + describe('R5-1: hardlink whose linkname is a symlink (extractFile)', () => { + it('R5-1: rejects hardlink with symlink as link source', async () => { + const dest = path.join(tempDir, 'dest'); + await fs.mkdir(dest); + + const externalDir = path.join(tempDir, 'external'); + await fs.mkdir(externalDir); + const secretFile = path.join(externalDir, 'secret'); + await fs.writeFile(secretFile, 'sensitive'); + + // Step 1: plant symlink 's' → ../external/secret + // Step 2: hardlink 'myhardlink' with linkname 's' (the symlink) + const tar = buildTar([ + { name: 's', type: TarEntryType.SYMLINK, linkname: '../external/secret' }, + { name: 'myhardlink', type: TarEntryType.HARDLINK, linkname: 's' }, + ]); + const archive = path.join(tempDir, 'hl-symlink-src.tar.xz'); + await saveAsXz(tar, archive); + + await expect(extractFile(archive, { cwd: dest })).rejects.toThrow(/symlink/i); + + // myhardlink must not appear in dest + expect(await fs.readdir(dest)).not.toContain('myhardlink'); + + // External secret file's link count must remain 1 (hardlink not created) + const secretStat = await fs.stat(secretFile); + expect(secretStat.nlink).toBe(1); + }); + }); + + // ------------------------------------------------------------------------- + // S2 — Hardlink escape (extractFile) + // ------------------------------------------------------------------------- + + describe('S2: hardlink escape via relative ".." linkname (extractFile)', () => { + it('S2: rejects hardlink whose linkname resolves outside cwd', async () => { + const dest = path.join(tempDir, 'dest'); + await fs.mkdir(dest); + + // HARDLINK with linkname '../external/target' escapes cwd + const tar = buildTar([ + { name: 'escape-link', type: TarEntryType.HARDLINK, linkname: '../external/target' }, + ]); + const archive = path.join(tempDir, 'hl-escape.tar.xz'); + await saveAsXz(tar, archive); + + await expect(extractFile(archive, { cwd: dest })).rejects.toThrow(/cwd/i); + }); + }); + + // ------------------------------------------------------------------------- + // V6a/V6b — NUL byte / empty name (ensureSafeName) + // ------------------------------------------------------------------------- + + describe('V6a/V6b: NUL byte / empty name (ensureSafeName)', () => { + it('V6a: rejects SYMLINK entry with empty linkname', async () => { + const tar = buildTar([{ name: 'link.txt', type: TarEntryType.SYMLINK, linkname: '' }]); + const archive = path.join(tempDir, 'empty-linkname.tar.xz'); + await saveAsXz(tar, archive); + + const dest = path.join(tempDir, 'dest'); + await fs.mkdir(dest); + + await expect(extractFile(archive, { cwd: dest })).rejects.toThrow(/empty linkname/i); + }); + + it('V6b: rejects SYMLINK with NUL byte in linkname', async () => { + // Craft a SYMLINK header with NUL at offset 158 (inside linkname field) + const header = createHeader({ + name: 'link.txt', + type: TarEntryType.SYMLINK, + linkname: 'target.txt', + }); + // Inject NUL at position 158 — linkname field starts at 157 + header[158] = 0x00; + // Recalculate checksum + let checksum = 0; + for (let i = 0; i < 512; i++) { + checksum += i >= 148 && i < 156 ? 0x20 : header[i]!; + } + const checksumStr = checksum.toString(8).padStart(6, '0') + '\x00 '; + for (let i = 0; i < 8; i++) header[148 + i] = checksumStr.charCodeAt(i); + + const archive = path.join(tempDir, 'nul-linkname.tar.xz'); + await saveAsXz( + Buffer.concat([Buffer.from(header), Buffer.from(createEndOfArchive())]), + archive + ); + + const dest = path.join(tempDir, 'dest'); + await fs.mkdir(dest); + + // NUL truncates linkname at position 1 ('t') — result is '' which is empty, + // OR the NUL-containing string is caught by NUL check. Either is safe. + // Key: no file escapes dest. + try { + await extractFile(archive, { cwd: dest }); + const destContents = await fs.readdir(dest); + for (const f of destContents) { + expect(path.resolve(dest, f).startsWith(dest)).toBe(true); + } + } catch { + // Rejection is acceptable — both outcomes are safe + } + }); + + it('V6b: rejects FILE entry name containing NUL byte', async () => { + // Craft a FILE header with NUL at offset 4 in the name field + const content = Buffer.from('evil'); + const header = createHeader({ name: 'safe.txt', size: content.length }); + header[4] = 0x00; + // Recalculate checksum + let checksum = 0; + for (let i = 0; i < 512; i++) { + checksum += i >= 148 && i < 156 ? 0x20 : header[i]!; + } + const checksumStr = checksum.toString(8).padStart(6, '0') + '\x00 '; + for (let i = 0; i < 8; i++) header[148 + i] = checksumStr.charCodeAt(i); + + const archive = path.join(tempDir, 'nul-name.tar.xz'); + await saveAsXz( + Buffer.concat([ + Buffer.from(header), + content, + Buffer.alloc(calculatePadding(content.length)), + Buffer.from(createEndOfArchive()), + ]), + archive + ); + + const dest = path.join(tempDir, 'dest'); + await fs.mkdir(dest); + + // NUL truncates name — result is safe extraction OR rejection. Both are safe. + try { + await extractFile(archive, { cwd: dest }); + const destContents = await fs.readdir(dest); + for (const f of destContents) { + expect(path.resolve(dest, f).startsWith(dest)).toBe(true); + } + } catch { + // Rejection is acceptable + } + }); + }); + + // ------------------------------------------------------------------------- + // V12 — setuid/setgid/sticky bit stripping (extractFile) + // ------------------------------------------------------------------------- + + describe('V12: setuid/setgid/sticky bit stripping (extractFile)', () => { + it('V12: strips setuid bit from extracted file (0o4755 → 0o755)', async () => { + const dest = path.join(tempDir, 'dest'); + await fs.mkdir(dest); + + const tar = buildTar([{ name: 'x', content: Buffer.from('data') }]); + const tarBuf = Buffer.from(tar); + // Patch mode field at offset 100 to 0o4755 (setuid + rwxr-xr-x) + const modeStr = (0o4755).toString(8).padStart(7, '0') + '\x00'; + for (let i = 0; i < 8; i++) tarBuf[100 + i] = modeStr.charCodeAt(i); + // Recalculate checksum + let checksum = 0; + for (let i = 0; i < 512; i++) { + checksum += i >= 148 && i < 156 ? 0x20 : tarBuf[i]!; + } + const checksumStr = checksum.toString(8).padStart(6, '0') + '\x00 '; + for (let i = 0; i < 8; i++) tarBuf[148 + i] = checksumStr.charCodeAt(i); + + const archive = path.join(tempDir, 'setuid.tar.xz'); + await saveAsXz(tarBuf, archive); + + await extractFile(archive, { cwd: dest }); + + const stat = await fs.stat(path.join(dest, 'x')); + expect(stat.mode & 0o7000).toBe(0); // setuid bit stripped + expect(stat.mode & 0o777).toBe(0o755); // rwxr-xr-x preserved + }); + + it('V12: strips setgid+sticky bits from extracted directory', async () => { + const dest = path.join(tempDir, 'dest'); + await fs.mkdir(dest); + + const header = createHeader({ name: 'testdir/', type: TarEntryType.DIRECTORY, mode: 0o3755 }); + const tarBuf = Buffer.concat([Buffer.from(header), Buffer.from(createEndOfArchive())]); + + const archive = path.join(tempDir, 'setgid-dir.tar.xz'); + await saveAsXz(tarBuf, archive); + + await extractFile(archive, { cwd: dest }); + + const stat = await fs.stat(path.join(dest, 'testdir')); + expect(stat.mode & 0o7000).toBe(0); // setgid+sticky stripped + }); + }); + + // ------------------------------------------------------------------------- + // V2/V3 — fd-based O_NOFOLLOW (extractFile, POSIX only) + // ------------------------------------------------------------------------- + + describe('V2/V3: fd-based O_NOFOLLOW extraction (POSIX only, extractFile)', () => { + it('V2/V3: O_NOFOLLOW catches symlink even if leaf check somehow missed it', async () => { + if (process.platform === 'win32') { + // S-14: Windows uses by-path extraction — O_NOFOLLOW unavailable. + // TOCTOU window scales with entry size on Windows (see §12.2 policy). + // This test is skipped; see TODO [Win32] handle-based extraction. + console.log( + 'SKIP V2/V3 on win32: O_NOFOLLOW unavailable — Windows TOCTOU window documented in §12.2' + ); + return; + } + + const dest = path.join(tempDir, 'dest'); + await fs.mkdir(dest); + + const externalDir = path.join(tempDir, 'external'); + await fs.mkdir(externalDir); + const secretFile = path.join(externalDir, 'secret'); + await fs.writeFile(secretFile, 'sensitive'); + + // Archive: SYMLINK evil → ../external/secret, then FILE evil + // The leaf-symlink check (ensureSafeTarget) would normally catch this, + // but on POSIX the O_NOFOLLOW open also rejects it independently. + const tar = buildTar([ + { name: 'evil', type: TarEntryType.SYMLINK, linkname: '../external/secret' }, + { name: 'evil', content: Buffer.from('overwrite attempt') }, + ]); + const archive = path.join(tempDir, 'onofollow.tar.xz'); + await saveAsXz(tar, archive); + + await expect(extractFile(archive, { cwd: dest })).rejects.toThrow(/symlink/i); + + // External secret must NOT be overwritten + expect(await fs.readFile(secretFile, 'utf8')).toBe('sensitive'); + }); + }); + + // ------------------------------------------------------------------------- + // S-14 — Windows TOCTOU window policy (threat-model documentation) + // ------------------------------------------------------------------------- + + describe('S-14: Windows TOCTOU window policy', () => { + it('S-14: on Windows, by-path extraction means TOCTOU window scales with entry size', async () => { + if (process.platform !== 'win32') { + // POSIX uses O_NOFOLLOW fd-based extraction — TOCTOU window is minimal. + // This test documents the Windows threat model; there is no runtime assertion + // for POSIX because the window is architectural, not runtime-observable. + console.log( + 'S-14 INFO (non-Windows): POSIX path uses O_NOFOLLOW — TOCTOU window minimal. ' + + 'Windows by-path fallback has extended window (see §12.2, TODO [Win32] handle-based extraction).' + ); + return; + } + + // On Windows: verify ensureSafeTarget runs ONCE before any write. + // We can only check that extraction completes or rejects safely — + // the window itself is architectural (by-path) and cannot be closed + // without handle-based extraction (CreateFileW + FILE_FLAG_OPEN_REPARSE_POINT). + // TODO [Win32] handle-based extraction for tar-xz Node extractFile. + const dest = path.join(tempDir, 'dest'); + await fs.mkdir(dest); + + const archive = path.join(tempDir, 'safe.tar.xz'); + const tar = buildTar([{ name: 'data.bin', content: Buffer.alloc(512, 0xab) }]); + await saveAsXz(tar, archive); + + // Should extract cleanly — no malicious entry in this archive + await expect(extractFile(archive, { cwd: dest })).resolves.toBeUndefined(); + expect(await fs.readFile(path.join(dest, 'data.bin'))).toHaveLength(512); + }); + }); + + // ------------------------------------------------------------------------- + // S-15 — PAX bomb DoS (parseTar MAX_PAX_HEADER_BYTES guard) + // ------------------------------------------------------------------------- + + describe('S-15: PAX bomb DoS — MAX_PAX_HEADER_BYTES guard', () => { + it('S-15a: PAX header claiming 2 GB but only 1 KB actual — stream ends safely (no 2 GB allocation)', async () => { + // The archive provides only 1 KB of PAX data after a header claiming 2 GB. + // The stream runs out before the PAX bomb guard fires, so parseTar throws + // "Unexpected end of archive" — which is also safe (no unbounded allocation). + // This verifies the memory safety property: the parser never allocates 2 GB. + const claimedSize = 2 * 1024 * 1024 * 1024; // 2 GB + const actualBytes = 1024; // only 1 KB provided + + const tarBuf = buildPaxBombTar(claimedSize, actualBytes); + const archive = path.join(tempDir, 'pax-bomb-truncated.tar.xz'); + await saveAsXz(tarBuf, archive); + + // Memory snapshot before: ensures we don't allocate 2 GB + const memBefore = process.memoryUsage(); + + let caught: Error | null = null; + try { + for await (const _ of extract(createReadStream(archive))) { + // drain + } + } catch (err) { + caught = err as Error; + } + + const memAfter = process.memoryUsage(); + const heapDelta = memAfter.heapUsed - memBefore.heapUsed; + + // Must throw (either "Unexpected end" for truncated, or TAR_PARSER_INVARIANT if + // the guard fires first — both are correct rejection outcomes) + expect(caught).not.toBeNull(); + const isExpectedError = + caught!.message.includes('Unexpected end') || + caught!.message.toLowerCase().includes('pax') || + (caught as Error & { code?: string }).code === 'TAR_PARSER_INVARIANT'; + expect(isExpectedError).toBe(true); + + // Heap delta must be < 5 MB — the parser must NOT have allocated 2 GB + const MAX_ALLOWED_BYTES = 5 * 1024 * 1024; // 5 MB + expect(heapDelta).toBeLessThan(MAX_ALLOWED_BYTES); + }); + + it('S-15b: PAX header providing > 1 MB of actual data triggers TAR_PARSER_INVARIANT', async () => { + // This test provides MORE than 1 MB of actual PAX data to trigger the guard. + // The guard fires in pullChunk() when state.buffer.length + chunk.length > MAX_PAX_HEADER_BYTES. + // Expected: throws with err.code === 'TAR_PARSER_INVARIANT' and message containing 'PAX'. + const OVER_LIMIT = 1024 * 1024 + 512; // 1 MB + 512 bytes — clearly over + const tarBuf = buildPaxBombTar(OVER_LIMIT, OVER_LIMIT); + const archive = path.join(tempDir, 'pax-over-limit.tar.xz'); + await saveAsXz(tarBuf, archive); + + let caught: Error | null = null; + try { + for await (const _ of extract(createReadStream(archive))) { + /* drain */ + } + } catch (err) { + caught = err as Error; + } + + expect(caught).not.toBeNull(); + expect(caught!.message).toMatch(/PAX|header exceeds/i); + expect((caught as Error & { code?: string }).code).toBe('TAR_PARSER_INVARIANT'); + }); + }); +}); diff --git a/packages/tar-xz/test/tar-parser-stream.spec.ts b/packages/tar-xz/test/tar-parser-stream.spec.ts new file mode 100644 index 0000000..6815f88 --- /dev/null +++ b/packages/tar-xz/test/tar-parser-stream.spec.ts @@ -0,0 +1,319 @@ +/** + * Unit tests for parseTar() streaming TAR parser generator. + * Story: TAR-XZ-STREAMING-2026-04-28, Block 2 + * + * Test layout: flat (alongside coverage.spec.ts) per spec §12.5 / L-L-08. + */ +import { describe, expect, it } from 'vitest'; +import { calculatePadding, createEndOfArchive, createHeader } from '../src/tar/format.js'; +import { createPaxHeaderBlocks } from '../src/tar/pax.js'; +import { type ParseEvent, parseTar } from '../src/node/tar-parser.js'; +import { TarEntryType } from '../src/types.js'; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: test helper covering PAX/regular/global entries +/** Build a raw TAR buffer with end-of-archive blocks. */ +function buildTar( + entries: Array<{ + name: string; + content?: Buffer; + type?: string; + linkname?: string; + usePax?: boolean; + globalPax?: boolean; + globalAttrs?: Record; + }> +): Buffer { + const blocks: Buffer[] = []; + + for (const entry of entries) { + const content = entry.content ?? Buffer.alloc(0); + const type = (entry.type ?? TarEntryType.FILE) as string; + const isDir = type === TarEntryType.DIRECTORY; + const isLink = type === TarEntryType.SYMLINK || type === TarEntryType.HARDLINK; + const size = isDir || isLink ? 0 : content.length; + + if (entry.globalPax && entry.globalAttrs) { + // Build a PAX_GLOBAL block manually. + const attrs = entry.globalAttrs; + const paxData = Object.entries(attrs) + .map(([k, v]) => { + const line = ` ${k}=${v}\n`; + const len = String(line.length + 1).length + line.length; + return `${len}${line}`; + }) + .join(''); + const paxBuf = Buffer.from(paxData); + const paxPad = calculatePadding(paxBuf.length); + const gHeader = createHeader({ + name: '././@PaxHeader', + size: paxBuf.length, + type: 'g' as '0', + }); + blocks.push(Buffer.from(gHeader)); + blocks.push(paxBuf); + if (paxPad > 0) blocks.push(Buffer.alloc(paxPad)); + continue; + } + + let headerName = entry.name; + + if (entry.usePax || headerName.length > 99) { + const paxBlocks = createPaxHeaderBlocks(headerName, { + path: headerName, + size, + linkpath: entry.linkname, + }); + for (const block of paxBlocks) { + blocks.push(Buffer.from(block)); + } + headerName = headerName.slice(-99); + } + + const header = createHeader({ + name: headerName, + size, + type: type as '0', + linkname: entry.linkname, + }); + blocks.push(Buffer.from(header)); + + if (size > 0) { + blocks.push(content); + const pad = calculatePadding(size); + if (pad > 0) blocks.push(Buffer.alloc(pad)); + } + } + + blocks.push(Buffer.from(createEndOfArchive())); + return Buffer.concat(blocks); +} + +/** Build a raw TAR buffer WITHOUT end-of-archive (truncated). */ +function buildTarTruncated(entry: { name: string; content: Buffer }): Buffer { + const header = createHeader({ + name: entry.name, + size: entry.content.length, + type: '0', + }); + // Only half the content — simulates truncated archive. + return Buffer.concat([ + Buffer.from(header), + entry.content.subarray(0, Math.ceil(entry.content.length / 2)), + ]); +} + +/** Chunk `buf` into `chunkSize`-byte pieces. */ +function* chunkBuffer(buf: Buffer, chunkSize: number): Generator { + let offset = 0; + while (offset < buf.length) { + const end = Math.min(offset + chunkSize, buf.length); + yield new Uint8Array(buf.buffer, buf.byteOffset + offset, end - offset); + offset = end; + } +} + +/** Async iterable from a sync generator of Uint8Array. */ +async function* asyncChunks(gen: Iterable): AsyncIterable { + for (const chunk of gen) { + yield chunk; + } +} + +/** Collect all ParseEvents from parseTar into an array. */ +async function collectEvents( + source: AsyncIterable, + mode: 'extract' | 'list' +): Promise { + const events: ParseEvent[] = []; + for await (const ev of parseTar(source, mode)) { + events.push(ev); + } + return events; +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +describe('parseTar() — streaming TAR parser', () => { + it('3-entry archive fed in 128-byte chunks — all entries parsed correctly', async () => { + const tar = buildTar([ + { name: 'a.txt', content: Buffer.from('hello') }, + { name: 'b.txt', content: Buffer.from('world-world-world') }, + { name: 'c.txt', content: Buffer.from('!') }, + ]); + + const source = asyncChunks(chunkBuffer(tar, 128)); + const events = await collectEvents(source, 'extract'); + + // Expect: entry a, chunks for a, entry b, chunks for b, entry c, chunks for c, end + const entries = events.filter((e) => e.kind === 'entry'); + expect(entries).toHaveLength(3); + expect((entries[0] as Extract).entry.name).toBe('a.txt'); + expect((entries[1] as Extract).entry.name).toBe('b.txt'); + expect((entries[2] as Extract).entry.name).toBe('c.txt'); + + // Collect chunks per entry + function chunksFor(entryIndex: number): Buffer { + const parts: Buffer[] = []; + let idx = 0; + let targetEntry = -1; + for (const ev of events) { + if (ev.kind === 'entry') { + idx++; + targetEntry = idx - 1; + } else if (ev.kind === 'chunk' && targetEntry === entryIndex) { + parts.push(Buffer.from(ev.data)); + } + } + return Buffer.concat(parts); + } + + expect(chunksFor(0).toString()).toBe('hello'); + expect(chunksFor(1).toString()).toBe('world-world-world'); + expect(chunksFor(2).toString()).toBe('!'); + + const endEvents = events.filter((e) => e.kind === 'end'); + expect(endEvents).toHaveLength(1); + }); + + it('PAX header payload split across 513-byte chunks (S-05) — entry.name correctly assembled', async () => { + // Create an entry with a long name that forces PAX usage. + const longName = `${'a'.repeat(150)}/file.txt`; + const content = Buffer.from('pax-content'); + const tar = buildTar([{ name: longName, content }]); + + // 513 bytes forces splits at 512+1 boundary, exercising consumePaxHeader re-entrancy. + const source = asyncChunks(chunkBuffer(tar, 513)); + const events = await collectEvents(source, 'extract'); + + const entryEvents = events.filter((e) => e.kind === 'entry'); + expect(entryEvents).toHaveLength(1); + expect((entryEvents[0] as Extract).entry.name).toBe(longName); + + const chunkEvents = events.filter((e) => e.kind === 'chunk'); + const assembled = Buffer.concat( + chunkEvents.map((e) => Buffer.from((e as Extract).data)) + ); + expect(assembled.toString()).toBe('pax-content'); + }); + + it('PAX_GLOBAL header split across chunks (L-M-03) — globals correctly applied', async () => { + // Build: PAX_GLOBAL (comment attr) then a regular entry. + const tar = buildTar([ + { + name: 'global-entry', + globalPax: true, + globalAttrs: { comment: 'global-comment' }, + }, + { name: 'real.txt', content: Buffer.from('real') }, + ]); + + // Small chunks force the PAX_GLOBAL split. + const source = asyncChunks(chunkBuffer(tar, 128)); + const events = await collectEvents(source, 'extract'); + + const entryEvents = events.filter((e) => e.kind === 'entry'); + // The PAX_GLOBAL entry is consumed internally — only the real entry is emitted. + expect(entryEvents).toHaveLength(1); + expect((entryEvents[0] as Extract).entry.name).toBe('real.txt'); + }); + + it('two consecutive empty blocks → emits kind:end', async () => { + // Create minimal archive (just EOA). + const tar = Buffer.from(createEndOfArchive()); + const source = asyncChunks(chunkBuffer(tar, 512)); + const events = await collectEvents(source, 'extract'); + + expect(events).toHaveLength(1); + expect(events[0]?.kind).toBe('end'); + }); + + it('truncated stream → throws "Unexpected end of archive" before kind:end (S-09)', async () => { + const tar = buildTarTruncated({ + name: 'big.bin', + content: Buffer.alloc(1024, 0xab), + }); + + const source = asyncChunks(chunkBuffer(tar, 128)); + await expect(collectEvents(source, 'extract')).rejects.toThrow('Unexpected end of archive'); + }); + + it('mode:list never emits kind:chunk events', async () => { + const tar = buildTar([ + { name: 'x.bin', content: Buffer.alloc(4096, 0xff) }, + { name: 'y.bin', content: Buffer.alloc(2048, 0xee) }, + ]); + + const source = asyncChunks(chunkBuffer(tar, 256)); + const events = await collectEvents(source, 'list'); + + const chunkEvents = events.filter((e) => e.kind === 'chunk'); + expect(chunkEvents).toHaveLength(0); + + const entryEvents = events.filter((e) => e.kind === 'entry'); + expect(entryEvents).toHaveLength(2); + expect(events.at(-1)?.kind).toBe('end'); + }); + + it('auto-drain (S-08): consumer skips entry.data — parser silently consumes content + padding before next header', async () => { + // Build archive with large first entry; consumer will skip it. + const tar = buildTar([ + { name: 'skip-me.bin', content: Buffer.alloc(1024, 0xaa) }, + { name: 'read-me.txt', content: Buffer.from('correct') }, + ]); + + const source = asyncChunks(chunkBuffer(tar, 128)); + const parser = parseTar(source, 'extract'); + + // Pull the first entry but consume NO chunks. + const first = await parser.next(); + expect(first.done).toBe(false); + expect((first.value as Extract).entry.name).toBe('skip-me.bin'); + + // Immediately advance to next entry — must skip over chunks. + // parseTar emits chunks before the next 'entry' event, so drain manually. + let ev: IteratorResult; + do { + ev = await parser.next(); + } while (!ev.done && ev.value.kind === 'chunk'); + + // Should now be at 'entry' for read-me.txt + expect(ev.done).toBe(false); + expect((ev.value as Extract).kind).toBe('entry'); + expect((ev.value as Extract).entry.name).toBe('read-me.txt'); + + // Drain the content for read-me.txt. + const parts: Buffer[] = []; + for await (const event of parser) { + if (event.kind === 'chunk') parts.push(Buffer.from(event.data)); + if (event.kind === 'end') break; + } + expect(Buffer.concat(parts).toString()).toBe('correct'); + }); + + it('parser invariant violation throws error with code TAR_PARSER_INVARIANT (D-5 / L-S-02)', async () => { + // Manufacture a stream that triggers the PAX bomb guard (> 1 MB in header phase). + const MAX_PAX = 1024 * 1024; + // Create a source that yields a chunk larger than MAX_PAX during header phase. + async function* bigSource(): AsyncIterable { + yield new Uint8Array(MAX_PAX + 1); + } + + let caught: Error | undefined; + try { + for await (const _ of parseTar(bigSource(), 'extract')) { + // nothing + } + } catch (e) { + caught = e as Error; + } + + expect(caught).toBeDefined(); + expect((caught as Error & { code?: string }).code).toBe('TAR_PARSER_INVARIANT'); + }); +}); diff --git a/packages/tar-xz/test/xz-helpers.spec.ts b/packages/tar-xz/test/xz-helpers.spec.ts new file mode 100644 index 0000000..eaf8b30 --- /dev/null +++ b/packages/tar-xz/test/xz-helpers.spec.ts @@ -0,0 +1,284 @@ +/** + * Tests for streamXz() and the deprecated helpers in xz-helpers.ts + * Story: TAR-XZ-STREAMING-2026-04-28, Block 1 + * + * Test layout: flat (alongside coverage.spec.ts, node-api.spec.ts, tar-format.spec.ts) + * per spec §12.5 / L-L-08 (existing flat layout convention). + */ +import { Readable } from 'node:stream'; +import { xzSync } from 'node-liblzma'; +import { describe, expect, it } from 'vitest'; +import { create } from '../src/node/create.js'; +import { streamXz } from '../src/node/xz-helpers.js'; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/** + * Build a tiny valid .tar.xz from a list of files using the real `create()` + * function. Returns the compressed bytes as a Buffer. + */ +async function buildTarXz( + files: Array<{ name: string; content: Uint8Array | Buffer }> +): Promise { + const chunks: Uint8Array[] = []; + for await (const chunk of create({ + files: files.map((f) => ({ + name: f.name, + source: Buffer.isBuffer(f.content) ? f.content : Buffer.from(f.content), + })), + })) { + chunks.push(chunk); + } + return Buffer.concat(chunks.map((c) => Buffer.from(c.buffer, c.byteOffset, c.byteLength))); +} + +/** Collect all chunks from an AsyncIterable into a flat Buffer. */ +async function collectIterable(iterable: AsyncIterable): Promise { + const parts: Uint8Array[] = []; + for await (const chunk of iterable) { + parts.push(chunk); + } + return Buffer.concat(parts.map((c) => Buffer.from(c.buffer, c.byteOffset, c.byteLength))); +} + +// --------------------------------------------------------------------------- +// T-07: Lazy pipeline — calling streamXz() without iterating must NOT start +// consuming the input Readable and must NOT produce unhandled rejections. +// --------------------------------------------------------------------------- + +/** + * Build a mock Readable that counts how many times _read() is called, so we + * can assert the pipeline never touched the source when the generator was + * returned before first .next(). + */ +function makeCountingReadable(data: Buffer): { readable: Readable; getReadCount: () => number } { + let readCount = 0; + let consumed = false; + const readable = new Readable({ + read() { + readCount++; + if (!consumed) { + consumed = true; + this.push(data); + this.push(null); + } + }, + }); + return { readable, getReadCount: () => readCount }; +} + +// --------------------------------------------------------------------------- +// T-01: streamXz(Uint8Array) decompresses correctly (full byte-equality) +// --------------------------------------------------------------------------- + +describe('streamXz', () => { + it('T-01: decompresses a Uint8Array input with byte equality', async () => { + const rawContent = Buffer.from('hello from tar-xz streaming test T-01'); + const compressed = xzSync(rawContent); + const input = new Uint8Array(compressed.buffer, compressed.byteOffset, compressed.byteLength); + + const result = await collectIterable(streamXz(input)); + + expect(result).toEqual(rawContent); + }); + + // --------------------------------------------------------------------------- + // T-02: Multi-chunk yield — archive larger than one XZ output chunk + // --------------------------------------------------------------------------- + + it('T-02: yields MORE THAN ONE chunk for a large input (no pre-buffering)', async () => { + // Build a ~100 KB tar.xz so that the XZ decompressor emits multiple output chunks. + // Each XZ output buffer is typically 64 KB, so a ~100 KB payload forces ≥2 chunks. + const bigFile = Buffer.alloc(100 * 1024, 0x42); // 100 KB of 'B' + const tarXz = await buildTarXz([{ name: 'big.bin', content: bigFile }]); + + const chunks: Uint8Array[] = []; + let totalBytes = 0; + for await (const chunk of streamXz(tarXz)) { + chunks.push(chunk); + totalBytes += chunk.length; + } + + expect(chunks.length).toBeGreaterThan(1); + // Sum of chunk lengths must equal total decompressed size. + expect(totalBytes).toBe(chunks.reduce((s, c) => s + c.length, 0)); + // Concatenated bytes must be ≥ 100 KB (the file content plus TAR overhead). + expect(totalBytes).toBeGreaterThanOrEqual(bigFile.length); + }); + + // --------------------------------------------------------------------------- + // T-03: Input form variants — Uint8Array, Buffer, Readable, AsyncIterable + // --------------------------------------------------------------------------- + + it('T-03a: accepts Uint8Array input', async () => { + const data = Buffer.from('T-03a content'); + const compressed = xzSync(data); + const u8 = new Uint8Array(compressed.buffer, compressed.byteOffset, compressed.byteLength); + + const result = await collectIterable(streamXz(u8)); + expect(result).toEqual(data); + }); + + it('T-03b: accepts Buffer input', async () => { + const data = Buffer.from('T-03b content'); + const compressed = xzSync(data); + + const result = await collectIterable(streamXz(compressed)); + expect(result).toEqual(data); + }); + + it('T-03c: accepts Readable stream input', async () => { + const data = Buffer.from('T-03c content'); + const compressed = xzSync(data); + const readable = Readable.from( + (async function* () { + yield compressed; + })() + ); + + const result = await collectIterable(streamXz(readable)); + expect(result).toEqual(data); + }); + + it('T-03d: accepts AsyncIterable input', async () => { + const data = Buffer.from('T-03d content'); + const compressed = xzSync(data); + const asyncIterable: AsyncIterable = (async function* () { + yield new Uint8Array(compressed.buffer, compressed.byteOffset, compressed.byteLength); + })(); + + const result = await collectIterable(streamXz(asyncIterable)); + expect(result).toEqual(data); + }); + + // --------------------------------------------------------------------------- + // T-04: Corrupt input → thrown error in for await + // --------------------------------------------------------------------------- + + it('T-04: throws an error when the XZ input is corrupt', async () => { + const corrupt = Buffer.from('this is not valid XZ data at all!!! abcdef1234567890'); + + await expect(async () => { + await collectIterable(streamXz(corrupt)); + }).rejects.toThrow(); + }); + + // --------------------------------------------------------------------------- + // T-05: Memory shape — in-loop high-water mark sampling (spec §12.3 pattern) + // + // Uses a 5 MB synthetic decompressed tar and asserts that peak memory delta + // stays below 10× the typical XZ output chunk size (64 KB = 65,536 bytes). + // This proves no full-archive accumulation (which would spike to ~5 MB+). + // --------------------------------------------------------------------------- + + // --------------------------------------------------------------------------- + // T-06: Early termination — no unhandled rejections, stream destroyed + // --------------------------------------------------------------------------- + + it('T-06: breaking out of for-await early causes no unhandled rejection and stream is cleaned up', async () => { + // Build a 3-entry archive so there are entries past the break point. + const tarXz = await buildTarXz([ + { name: 'a.txt', content: Buffer.from('entry-a') }, + { name: 'b.txt', content: Buffer.from('entry-b') }, + { name: 'c.txt', content: Buffer.from('entry-c') }, + ]); + + const unhandledRejections: Error[] = []; + const handler = (err: Error) => unhandledRejections.push(err); + process.on('unhandledRejection', handler); + + try { + // Only consume first chunk — break immediately + let count = 0; + for await (const _chunk of streamXz(tarXz)) { + count++; + break; // consumer breaks early + } + // Allow a microtask turn for any unhandled rejection to surface + await new Promise((r) => setTimeout(r, 20)); + } finally { + process.off('unhandledRejection', handler); + } + + expect(unhandledRejections).toHaveLength(0); + }); + + it('T-05: memory shape — peak heapUsed delta stays below 2× decompressed size for 5 MB tar', async () => { + // Build a 5 MB synthetic file (highly compressible uniform byte) and stream-decompress it. + // The compressed archive is tiny (high ratio), so holding it in memory as input is cheap. + // The key proof: peak delta must stay well below the full decompressed size (5 MB), + // demonstrating that bytes are not accumulated before yielding. + // + // Threshold = 2 × payloadSize (10 MB): + // - A Buffer.concat-everything accumulator would spike to ≥ payloadSize (5 MB) just for + // the output array, PLUS the input buffer simultaneously, reaching 10+ MB. + // - A streaming pipeline drains chunks as fast as they arrive; peak is bounded by the + // largest transient buffer (one XZ output chunk ≈ 64 KB) plus pipeline internals. + // - The 2× slack covers GC timing variance, Node internal stream buffers, and + // the decompressor's own sliding-window dictionary (~8 MB for preset 6, shared + // across the whole decompression call but released on stream end). + // + // Note: this test uses `process.memoryUsage()` polling without `--expose-gc`. + // Full deterministic memory shape tests with explicit GC are in Block 5 + // (test/memory-shape.spec.ts with pool: 'forks' + execArgv: ['--expose-gc']). + const payloadSize = 5 * 1024 * 1024; // 5 MB + const payload = Buffer.alloc(payloadSize, 0xab); + const tarXz = await buildTarXz([{ name: 'data.bin', content: payload }]); + + const baseline = process.memoryUsage(); + const baselineSample = baseline.heapUsed + baseline.external; + let peak = baselineSample; + + const sample = () => { + const m = process.memoryUsage(); + peak = Math.max(peak, m.heapUsed + m.external); + }; + + let totalBytes = 0; + for await (const chunk of streamXz(tarXz)) { + sample(); + totalBytes += chunk.length; + } + sample(); + + const peakDelta = peak - baselineSample; + const threshold = 2 * payloadSize; // 10 MB + expect(peakDelta).toBeLessThan(threshold); + // Sanity: decompressed bytes must cover the full payload (TAR overhead adds more). + expect(totalBytes).toBeGreaterThanOrEqual(payloadSize); + }, 30_000); // 30s timeout — compression of 5 MB takes a few seconds + + // --------------------------------------------------------------------------- + // T-07: Lazy pipeline — pipeline must NOT start until first .next() call + // --------------------------------------------------------------------------- + + it('T-07: pipeline does not start when the returned AsyncIterable is never iterated', async () => { + // Use a counting Readable so we can detect whether _read() was ever called. + // We give it valid XZ data (so if the pipeline DID start it would succeed), + // making read-count the only observable signal of eagerness. + const data = Buffer.from('T-07 lazy semantics check'); + const compressed = xzSync(data); + const { readable, getReadCount } = makeCountingReadable(compressed); + + const unhandledRejections: Error[] = []; + const handler = (err: Error) => unhandledRejections.push(err); + process.on('unhandledRejection', handler); + + try { + // Obtain the iterable but do NOT iterate it. + const _iterable = streamXz(readable); + + // Allow a microtask turn — enough for any eagerly-started pipeline to tick. + await new Promise((r) => setTimeout(r, 20)); + } finally { + process.off('unhandledRejection', handler); + } + + // (a) Pipeline must NOT have consumed the input — readable._read() never called. + expect(getReadCount()).toBe(0); + // (b) No unhandled rejections — lazy pipeline means no dangling promise. + expect(unhandledRejections).toHaveLength(0); + }); +}); diff --git a/packages/tar-xz/vitest.config.ts b/packages/tar-xz/vitest.config.ts index f7d41a1..0a11582 100644 --- a/packages/tar-xz/vitest.config.ts +++ b/packages/tar-xz/vitest.config.ts @@ -2,6 +2,9 @@ import { defineConfig } from 'vitest/config'; export default defineConfig({ test: { + // Use threads pool (default) for fast parallel execution. + // Memory-shape tests that require --expose-gc have their own config: + // vitest.memory.config.ts (pool: 'forks') — run via pnpm test:memory. include: ['test/**/*.spec.ts'], coverage: { provider: 'v8', diff --git a/packages/tar-xz/vitest.memory.config.ts b/packages/tar-xz/vitest.memory.config.ts new file mode 100644 index 0000000..e9a734e --- /dev/null +++ b/packages/tar-xz/vitest.memory.config.ts @@ -0,0 +1,18 @@ +import { defineConfig } from 'vitest/config'; + +/** + * Separate Vitest config for memory-shape tests only. + * + * These tests require --expose-gc (available via Node forks pool) and are + * intentionally isolated here so the default vitest.config.ts can use the + * faster threads pool for the rest of the suite. + * + * Usage: pnpm test:memory + */ +export default defineConfig({ + test: { + pool: 'forks', + execArgv: ['--expose-gc'], + include: ['test/memory-shape.spec.ts'], + }, +});