feat(zig-cli): -C/--container — emit & consume .pbf.json (issue #1)

pmarreck · pmarreck · commit e76f6efb54df · 2026-06-27T09:20:32.000-04:00
The Zig CLI now understands the container: a hand-rolled flat-JSON envelope
(naturally transport-resistant — reads 'data' to its closing quote, then
canonicalizes) over the shared core crc32/encode/decode. '-C file' emits a
self-verifying .pbf.json; '-d -C' restores the original bytes.

Adds test-container-zig (the parameterized round-trip/self-verify/transport guard
against the Zig CLI) and test-container-cross (the MFIC differential: Node and Zig
containers decode each other byte-identically).
diff --git a/.dirtree-state b/.dirtree-state
@@ -6,5 +6,6 @@ annotate=[
 	src/zig/ffi.zig = C ABI (FFI) export surface: all 12 pb_* C functions; root of libprintable_binary.a; keeps C symbols OUT of the importable printable_binary module so static (musl) consumers don't collide
 	test/module_consumer.zig = Test fixture: minimal downstream importer of the printable_binary Zig module (mirrors how difz/blip consume it) for the FFI-symbol-leak test
 	test/test_container = Container (.pbf.json) CLI round-trip + self-verify test, parameterized by IMPLEMENTATION_TO_TEST (issue #1)
+	test/test_container_cross = Cross-impl container differential: impl-A container decodes via impl-B (MFIC, issue #1)
 	test/test_module_no_ffi_symbols = Regression test (nm oracle): importing the module must emit 0 pb_* symbols while the FFI static lib keeps all 12 — guards the difz-blocking duplicate-symbol bug
 ]
diff --git a/flake.nix b/flake.nix
@@ -337,6 +337,33 @@
             installPhase = "mkdir -p $out && touch $out/passed";
           };
 
+          # Container (.pbf.json) for the Zig CLI (issue #1).
+          test-container-zig = pkgs.stdenv.mkDerivation {
+            name = "test-container-zig";
+            src = ./.;
+            nativeBuildInputs = with pkgs; [ zig ];
+            buildPhase = ''
+              export HOME=$TMPDIR
+              zig build -Doptimize=ReleaseFast
+              IMPLEMENTATION_TO_TEST=./zig-out/bin/printable-binary-zig bash ./test/test_container
+            '';
+            installPhase = "mkdir -p $out && touch $out/passed";
+          };
+
+          # Cross-impl container differential: Node <-> Zig must agree (issue #1).
+          test-container-cross = pkgs.stdenv.mkDerivation {
+            name = "test-container-cross";
+            src = ./.;
+            nativeBuildInputs = with pkgs; [ zig nodejs_24 ];
+            buildPhase = ''
+              export HOME=$TMPDIR
+              patchShebangs bin/printable-binary-node.js
+              zig build -Doptimize=ReleaseFast
+              IMPL_A=./bin/printable-binary-node.js IMPL_B=./zig-out/bin/printable-binary-zig bash ./test/test_container_cross
+            '';
+            installPhase = "mkdir -p $out && touch $out/passed";
+          };
+
           # Dogfood the C FFI boundary: build the C FFI CLI against the Zig static
           # lib and round-trip through it (encode/decode + hexlike).
           test-ffi-cli = pkgs.stdenv.mkDerivation {
diff --git a/src/zig/main.zig b/src/zig/main.zig
@@ -29,6 +29,7 @@ const Options = struct {
     range_end: ?i64 = null,
     no_double_encode_check: bool = false,
     hexlike: bool = false,
+    container: bool = false,
 };
 
 const MappingsMode = enum { none, table, json, csv };
@@ -290,6 +291,8 @@ fn parseArgs(allocator: std.mem.Allocator, args: []const []const u8) !Options {
                 opts.no_double_encode_check = true;
             } else if (std.mem.eql(u8, name, "hexlike")) {
                 opts.hexlike = true;
+            } else if (std.mem.eql(u8, name, "container")) {
+                opts.container = true;
             } else if (std.mem.eql(u8, name, "help")) {
                 opts.help_mode = true;
             } else {
@@ -312,6 +315,7 @@ fn parseArgs(allocator: std.mem.Allocator, args: []const []const u8) !Options {
                     },
                     'S' => opts.strip_whitespace = true,
                     'X' => opts.hexlike = true,
+                    'C' => opts.container = true,
                     'h' => opts.help_mode = true,
                     'f' => {
                         if (j + 1 < arg.len) {
@@ -367,6 +371,8 @@ fn printUsage(io: std.Io) void {
         \\                     shown as uppercase hex runs prefixed by Οχ (Greek Omicron+Chi,
         \\                     NOT ASCII 0x — beware when copying hex for other purposes).
         \\                     Use with -d to decode hexlike-encoded data back to binary.
+        \\  -C, --container    Container mode: encode a file to a self-verifying .pbf.json
+        \\                     (keeps filename + crc32). With -d, decode a container back.
         \\
         \\Encode options (preserve literal characters instead of encoding):
         \\  -s, --spaces       Preserve literal spaces (don't encode to visible glyph)
@@ -587,6 +593,14 @@ pub fn main(init: std.process.Init) !void {
         input = raw_input[range.offset .. range.offset + range.length];
     }
 
+    if (opts.container) {
+        handleContainer(io, allocator, opts, input) catch |err| {
+            writeStats("Container error: {}\n", .{err});
+            std.process.exit(1);
+        };
+        return;
+    }
+
     if (opts.decode_mode) {
         // Decode mode - call core library
         if (opts.passthrough_mode) {
@@ -717,3 +731,161 @@ pub fn main(init: std.process.Init) !void {
         }
     }
 }
+
+// ============================================================================
+// Container (.pbf.json) support (issue #1)
+//
+// Hand-rolled flat-JSON, naturally transport-resistant: the `data` value is read
+// to its closing quote regardless of any whitespace a text transport injected
+// inside it, then canonicalized (whitespace stripped) before the crc check and
+// decode. crc32 is vector-pinned in the core (CRC32("123456789")=0xCBF43926), so
+// every implementation agrees. Architecture A2: the JSON envelope is assembled
+// here, the codec + crc32 come from the shared core.
+// ============================================================================
+
+/// Strip transport whitespace (space/tab/CR/LF) from an encoded payload. The
+/// default encoding emits none of these (all glyph'd), so a clean payload is
+/// unchanged; only transport-injected whitespace is removed.
+fn canonicalPayload(allocator: std.mem.Allocator, data: []const u8) ![]u8 {
+    var count: usize = 0;
+    for (data) |c| {
+        if (c != ' ' and c != '\t' and c != '\r' and c != '\n') count += 1;
+    }
+    const out = try allocator.alloc(u8, count);
+    var n: usize = 0;
+    for (data) |c| {
+        if (c != ' ' and c != '\t' and c != '\r' and c != '\n') {
+            out[n] = c;
+            n += 1;
+        }
+    }
+    return out;
+}
+
+/// Extract the raw string value of `key` from a flat JSON object (bytes between
+/// the quotes; no unescaping — our data/crc fields carry no escapes). Tolerant
+/// of surrounding whitespace. null if absent.
+fn jsonGetString(json: []const u8, key: []const u8) ?[]const u8 {
+    var keybuf: [64]u8 = undefined;
+    const needle = std.fmt.bufPrint(&keybuf, "\"{s}\"", .{key}) catch return null;
+    const kpos = std.mem.indexOf(u8, json, needle) orelse return null;
+    var i = kpos + needle.len;
+    while (i < json.len and (json[i] == ' ' or json[i] == '\t' or json[i] == '\r' or json[i] == '\n' or json[i] == ':')) : (i += 1) {}
+    if (i >= json.len or json[i] != '"') return null;
+    i += 1;
+    const start = i;
+    while (i < json.len) : (i += 1) {
+        if (json[i] == '\\') {
+            i += 1;
+            continue;
+        }
+        if (json[i] == '"') break;
+    }
+    if (i >= json.len) return null;
+    return json[start..i];
+}
+
+/// JSON-escape `s` into a freshly allocated buffer.
+fn jsonEscapeAlloc(allocator: std.mem.Allocator, s: []const u8) ![]u8 {
+    var extra: usize = 0;
+    for (s) |c| {
+        if (c == '"' or c == '\\' or c == '\n' or c == '\r' or c == '\t') extra += 1;
+    }
+    const out = try allocator.alloc(u8, s.len + extra);
+    var n: usize = 0;
+    for (s) |c| {
+        switch (c) {
+            '"' => {
+                out[n] = '\\';
+                out[n + 1] = '"';
+                n += 2;
+            },
+            '\\' => {
+                out[n] = '\\';
+                out[n + 1] = '\\';
+                n += 2;
+            },
+            '\n' => {
+                out[n] = '\\';
+                out[n + 1] = 'n';
+                n += 2;
+            },
+            '\r' => {
+                out[n] = '\\';
+                out[n + 1] = 'r';
+                n += 2;
+            },
+            '\t' => {
+                out[n] = '\\';
+                out[n + 1] = 't';
+                n += 2;
+            },
+            else => {
+                out[n] = c;
+                n += 1;
+            },
+        }
+    }
+    return out;
+}
+
+fn basenameOf(path: []const u8) []const u8 {
+    var start: usize = 0;
+    for (path, 0..) |c, idx| {
+        if (c == '/' or c == '\\') start = idx + 1;
+    }
+    return path[start..];
+}
+
+fn handleContainer(io: std.Io, allocator: std.mem.Allocator, opts: Options, input: []const u8) !void {
+    if (opts.decode_mode) {
+        const data_raw = jsonGetString(input, "data") orelse {
+            writeStats("Error: not a printable-binary-file container (missing 'data')\n", .{});
+            std.process.exit(1);
+        };
+        const clean = try canonicalPayload(allocator, data_raw);
+        defer allocator.free(clean);
+        if (jsonGetString(input, "crc32_encoded")) |want| {
+            var gb: [8]u8 = undefined;
+            const got = std.fmt.bufPrint(&gb, "{x:0>8}", .{pb.crc32(clean)}) catch unreachable;
+            if (!std.mem.eql(u8, got, want)) {
+                writeStats("Error: container crc32_encoded mismatch (data corrupted)\n", .{});
+                std.process.exit(1);
+            }
+        }
+        const decoded = pb.decode(allocator, clean, .{}) catch |err| {
+            writeStats("Container decode error: {}\n", .{err});
+            std.process.exit(1);
+        };
+        defer allocator.free(decoded);
+        if (jsonGetString(input, "crc32")) |want| {
+            var gb: [8]u8 = undefined;
+            const got = std.fmt.bufPrint(&gb, "{x:0>8}", .{pb.crc32(decoded)}) catch unreachable;
+            if (!std.mem.eql(u8, got, want)) {
+                writeStats("Error: container crc32 mismatch (decoded data corrupted)\n", .{});
+                std.process.exit(1);
+            }
+        }
+        writeStats("Decoded container: {d} bytes\n", .{decoded.len});
+        try writeOutput(io, decoded, false);
+    } else {
+        const data = pb.encode(allocator, input, .{}) catch |err| {
+            writeStats("Container encode error: {}\n", .{err});
+            std.process.exit(1);
+        };
+        defer allocator.free(data);
+        const clean = try canonicalPayload(allocator, data);
+        defer allocator.free(clean);
+        var ob: [8]u8 = undefined;
+        var eb: [8]u8 = undefined;
+        const crc_orig = std.fmt.bufPrint(&ob, "{x:0>8}", .{pb.crc32(input)}) catch unreachable;
+        const crc_enc = std.fmt.bufPrint(&eb, "{x:0>8}", .{pb.crc32(clean)}) catch unreachable;
+        const fname_in: []const u8 = if (opts.input_file) |p| (if (std.mem.eql(u8, p, "-")) "" else basenameOf(p)) else "";
+        const fname = try jsonEscapeAlloc(allocator, fname_in);
+        defer allocator.free(fname);
+        // `data` is LAST so all metadata sits up front.
+        const json = try std.fmt.allocPrint(allocator, "{{\n  \"format\": \"printable-binary-file\",\n  \"version\": 1,\n  \"filename\": \"{s}\",\n  \"byte_length\": {d},\n  \"crc32\": \"{s}\",\n  \"crc32_encoded\": \"{s}\",\n  \"data\": \"{s}\"\n}}\n", .{ fname, input.len, crc_orig, crc_enc, data });
+        defer allocator.free(json);
+        try writeOutput(io, json, false);
+    }
+}
diff --git a/test/test_container_cross b/test/test_container_cross
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Cross-impl container differential (MFIC): a container emitted by impl A must
+# decode byte-identically via impl B, and vice versa. External oracle against
+# any envelope/crc/encoding drift between implementations. (issue #1)
+set -u
+A="${IMPL_A:-./bin/printable-binary-node.js}"
+B="${IMPL_B:-./result/bin/printable-binary-zig}"
+TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT
+pass=0; fail=0
+ok()  { echo "  ✓ $1"; pass=$((pass+1)); }
+bad() { echo "  ✗ $1"; fail=$((fail+1)); }
+echo "=== cross-impl container differential: A=$A  B=$B ==="
+printf 'cross-impl \x00\x01\xff payload\n{"not":"json"}\ttabbed\n' > "$TMP/orig.bin"
+
+"$A" -C "$TMP/orig.bin" > "$TMP/a.pbf.json" 2>/dev/null
+"$B" -d -C "$TMP/a.pbf.json" > "$TMP/b_out.bin" 2>/dev/null
+cmp -s "$TMP/orig.bin" "$TMP/b_out.bin" && ok "A-encoded container decodes byte-identically via B" || bad "A-encoded container decodes byte-identically via B"
+
+"$B" -C "$TMP/orig.bin" > "$TMP/b.pbf.json" 2>/dev/null
+"$A" -d -C "$TMP/b.pbf.json" > "$TMP/a_out.bin" 2>/dev/null
+cmp -s "$TMP/orig.bin" "$TMP/a_out.bin" && ok "B-encoded container decodes byte-identically via A" || bad "B-encoded container decodes byte-identically via A"
+
+echo "cross-impl container [$A <-> $B]: $pass passed, $fail failed"
+[ "$fail" -eq 0 ]

Original file line number	Diff line number	Diff line change
`@@ -6,5 +6,6 @@ annotate=[`
`6`	`6`	`src/zig/ffi.zig = C ABI (FFI) export surface: all 12 pb_* C functions; root of libprintable_binary.a; keeps C symbols OUT of the importable printable_binary module so static (musl) consumers don't collide`
`7`	`7`	`test/module_consumer.zig = Test fixture: minimal downstream importer of the printable_binary Zig module (mirrors how difz/blip consume it) for the FFI-symbol-leak test`
`8`	`8`	`test/test_container = Container (.pbf.json) CLI round-trip + self-verify test, parameterized by IMPLEMENTATION_TO_TEST (issue #1)`
	`9`	`+ test/test_container_cross = Cross-impl container differential: impl-A container decodes via impl-B (MFIC, issue #1)`
`9`	`10`	`test/test_module_no_ffi_symbols = Regression test (nm oracle): importing the module must emit 0 pb_* symbols while the FFI static lib keeps all 12 — guards the difz-blocking duplicate-symbol bug`
`10`	`11`	`]`