Migrate to fixed node-pkware fork (v5-fix), drop streaming plumbing

Switches `node-pkware` from the 2021 fork commit to a new `v5-fix`
branch based on upstream `arx-tools/node-pkware` v5.9.1 with three
algorithmic bugs fixed (commit dacd10b on the fork):

  - module-level `lastOccurrences` map carrying stale offsets between
    Implode calls,
  - `getSizeOfMatching` capping the match at `b - a` for overlapping
    repetitions (was making run-length-encoding produce literal-per-byte
    output for repeating-byte runs),
  - `outputBits` overflow path not pre-allocating a fresh next byte
    when the spill exactly filled the new byte (caused corruption on
    the subsequent write).

ewd benefit: re-encoded `.ewprj` / `.ms14` are now ~2x the original
size instead of 7-10x. Round-trips remain byte-identical on every
sample tested.

Code change in this repo: `src/decode.ts` and `src/encode.ts` now use
the buffer-in/buffer-out `node-pkware/simple` API, drop the streaming
plumbing (Readable.from / pipe / streamToBuffer), and become
synchronous. New `src/util/buffer.ts` holds the Buffer -> ArrayBuffer
helper used by both.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

Author: cinderblock

README.md

@@ -182,12 +182,18 @@ untouched at that cell.
 ### Compression ratio caveat
 
 `ewe` uses [`node-pkware`](https://github.com/cinderblock/node-pkware)'s
-`implode` for compression. Its repetition search is currently incomplete
-(see the upstream `TODO: search for a better repetition` log lines), so
-re-encoded files are several times larger than the originals. They are
-still valid: a decode/encode/decode round-trip on every sample tested
-produces byte-identical XML, and the resulting files are accepted by
-`ewd`. Improving compression density is a `node-pkware` problem.
+`implode` for compression. The repetition search isn't quite as
+sophisticated as the original PKWare DCL implementation Multisim itself
+uses — re-encoded files are roughly **2× the size** of the originals
+(e.g. 9 KB `.ewprj` round-trips to 20 KB). The output is still valid
+PKWare DCL Implode: round-trips are byte-identical on every sample
+tested, and the resulting files are accepted by `ewd`.
+
+This is a substantial improvement over the previous baseline (7-10×
+bloat) thanks to fixes in the `v5-fix` branch of the
+`cinderblock/node-pkware` fork — see `plans/jet-in-place-edit.md`
+context and the fork's `v5-fix` branch for the specific bugs that
+were patched.
 
 ## Tests
 

bun.lock

@@ -23,7 +23,7 @@
   },
   "dependencies": {
     "command-line-args": "^6.0.2",
-    "node-pkware": "github:cinderblock/node-pkware",
+    "node-pkware": "cinderblock/node-pkware#v5-fix",
     "winston": "^3.19.0"
   },
   "devDependencies": {

package.json

@@ -1,23 +1,16 @@
 import { promises as fs } from 'node:fs';
-import { Readable } from 'node:stream';
-import { explode, stream } from 'node-pkware';
+import { explode } from 'node-pkware/simple';
 import type winston from 'winston';
 import { detectFormatByHeader, knownFormatsList, MAX_HEADER_LENGTH } from './formats';
+import { bufferToArrayBuffer } from './util/buffer';
 import { UnexpectedValue } from './util/UnexpectedValue';
 
-const { streamToBuffer, through } = stream;
-
-async function decodeBlock(block: Buffer, expectedLength: number): Promise<Buffer> {
-  const ret = await new Promise<Buffer>(resolve =>
-    Readable.from(block)
-      .pipe(through(explode({ inputBufferSize: block.length, outputBufferSize: expectedLength })))
-      .pipe(streamToBuffer(resolve)),
-  );
-
-  if (ret.length !== expectedLength)
-    throw new Error(`Decoder returned wrong length. Expected: ${expectedLength}. Got: ${ret.length}.`);
-
-  return ret;
+function decodeBlock(block: Buffer, expectedLength: number): Buffer {
+  const result = Buffer.from(explode(bufferToArrayBuffer(block)));
+  if (result.length !== expectedLength) {
+    throw new Error(`Decoder returned wrong length. Expected: ${expectedLength}. Got: ${result.length}.`);
+  }
+  return result;
 }
 
 export async function decode(filename: string, logger: winston.Logger, outFile = `${filename}.xml`): Promise<void> {
@@ -110,7 +103,7 @@ export async function decode(filename: string, logger: winston.Logger, outFile =
 
         decompressedBytesRead += length;
 
-        const decodedBlock = await decodeBlock(compressedData, length);
+        const decodedBlock = decodeBlock(compressedData, length);
 
         written += (await outputFile.write(decodedBlock)).bytesWritten;
       }

src/decode.ts

@@ -1,29 +1,15 @@
 import { promises as fs } from 'node:fs';
-import { Readable } from 'node:stream';
-import { constants, implode, stream } from 'node-pkware';
+import { implode } from 'node-pkware/simple';
 import type winston from 'winston';
 import { type EwbFormat, formatForExtension, knownFormatsList } from './formats';
-
-const { COMPRESSION_ASCII, DICTIONARY_SIZE_LARGE } = constants;
-const { streamToBuffer, through } = stream;
+import { bufferToArrayBuffer } from './util/buffer';
 
 // Decompressed bytes per block. Multisim/Ultiboard ship files chunked at this
 // boundary; matching it keeps the section layout identical to originals.
 export const DEFAULT_BLOCK_SIZE = 900000;
 
-function compressBlock(block: Buffer): Promise<Buffer> {
-  return new Promise<Buffer>((resolve, reject) => {
-    const src = Readable.from(block);
-    const compressor = through(
-      implode(COMPRESSION_ASCII, DICTIONARY_SIZE_LARGE, {
-        inputBufferSize: block.length,
-        outputBufferSize: block.length,
-      }),
-    );
-    src.on('error', reject);
-    compressor.on('error', reject);
-    src.pipe(compressor).pipe(streamToBuffer(resolve));
-  });
+function compressBlock(block: Buffer): Buffer {
+  return Buffer.from(implode(bufferToArrayBuffer(block), 'ascii', 'large'));
 }
 
 /**
@@ -95,7 +81,7 @@ export async function encode(inFile: string, logger: winston.Logger, options: En
       const end = Math.min(offset + blockSize, totalLength);
       const slice = xml.subarray(offset, end);
 
-      const compressed = await compressBlock(slice);
+      const compressed = compressBlock(slice);
 
       logger.silly(`Section #${section}: ${slice.length} bytes -> ${compressed.length} compressed`);
 

src/encode.ts

@@ -0,0 +1,8 @@
+/**
+ * Convert a Node `Buffer` to a standalone `ArrayBuffer` containing exactly
+ * the bytes of the Buffer. Buffer instances are often views onto a larger
+ * shared pool, so we slice to the relevant range.
+ */
+export function bufferToArrayBuffer(buf: Buffer): ArrayBuffer {
+  return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength) as ArrayBuffer;
+}