fix(serialization): pass the official protobuf conformance suite (binary)

Ran the official protobuf conformance_test_runner (v29.3) against a testee
backed by our lib. It surfaced real bugs, now fixed:

- parser: negative numbers (e.g. negative enum values) split as `-` + digits
  were rejected; handle a leading sign in numbers and option values. Extension
  ranges / `extend` blocks (valid in editions) are now skipped, not rejected.
- reader: `this.pos += this.uint32()` captured pos *before* uint32 advanced it,
  so unknown length-delimited fields were under-skipped by one byte and desynced
  the stream. Skip group wire types (3/4). Bounds-check every read and reject
  overlong (>10-byte) varints and truncated input (premature EOF) — previously
  malformed input was silently accepted.
- decode: repeated occurrences of a singular message field now MERGE (proto
  spec) instead of overwriting; reject field number 0; map message values
  default to an empty message when the entry omits the value.

Result: 2060 binary successes, 0 failures across proto3 + edition 2023
(JSON/JSPB/text-format/proto2 are skipped — out of scope). Added unit
regressions for each fix; conformance-testee.mjs committed as the harness
(needs PB_SRC pointing at a protobuf checkout). bench/protobuf-ours.mjs compares
our decode/encode vs protobuf-es.

Author: Thanga-Ganapathy

CHANGELOG.md

@@ -15,7 +15,10 @@ pre-`0.1.0` and the public API is unstable.
   `schema.parse(messageName, bytes)` / `schema.build(messageName, value)` decode
   and encode the binary wire format. Decoded objects use camelCase keys, BigInt
   for 64-bit ints, enum value-names, and `Uint8Array` for bytes; unknown fields
-  are preserved across re-encode. Byte-for-byte verified against protobuf-es.
+  are preserved across re-encode. Passes the **official protobuf conformance
+  suite** for binary wire format (2060 successes, 0 failures across proto3 +
+  edition 2023; JSON/text-format/proto2 out of scope); byte-for-byte verified
+  against protobuf-es.
 
 ### Changed
 

bench/protobuf-ours.mjs

@@ -0,0 +1,46 @@
+// Decode throughput: our reflective Protobuf (runtime:serialization) vs the
+// reference protobuf-es, on identical bytes. Runs in esrun (has runtime:
+// serialization); protobuf-es also runs for an in-engine baseline.
+import { create, toBinary, fromBinary } from "@bufbuild/protobuf";
+import { CatalogSchema } from "./gen/catalog_pb.js";
+
+const { Protobuf } = await import("runtime:serialization");
+const proto = `
+  syntax = "proto3";
+  package test;
+  message Book {
+    string id = 1; string author = 2; string title = 3; string genre = 4;
+    double price = 5; string publish_date = 6; string description = 7;
+  }
+  message Catalog { repeated Book catalog = 1; }
+`;
+const ours = new Protobuf.Schema(proto);
+
+const obj = { catalog: [] };
+for (let i = 0; i < 50000; i++) {
+  obj.catalog.push({
+    id: `bk${i}`, author: "Gambardella, Matthew", title: "XML Developer's Guide",
+    genre: "Computer", price: 44.95, publishDate: "2000-10-01",
+    description: "An in-depth look at creating applications with XML.",
+  });
+}
+const bytes = toBinary(CatalogSchema, create(CatalogSchema, obj));
+console.log(`payload: ${bytes.length} bytes, 50000 books`);
+
+function bench(name, fn, iters = 30) {
+  for (let i = 0; i < 5; i++) fn();
+  const t0 = performance.now();
+  for (let i = 0; i < iters; i++) fn();
+  const t1 = performance.now();
+  console.log(`${name.padEnd(28)}: ${((t1 - t0) / iters).toFixed(1)} ms/op`);
+}
+
+// correctness sanity
+const a = ours.parse("test.Catalog", bytes);
+console.log(`ours decoded ${a.catalog.length} books, first.price=${a.catalog[0].price}`);
+
+const esMsg = create(CatalogSchema, obj);
+bench("protobuf-es decode", () => fromBinary(CatalogSchema, bytes));
+bench("ours decode (parse)", () => ours.parse("test.Catalog", bytes));
+bench("protobuf-es encode", () => toBinary(CatalogSchema, esMsg));
+bench("ours encode (build)", () => ours.build("test.Catalog", obj));

crates/runtime/js/conformance-testee.mjs

@@ -0,0 +1,108 @@
+// Conformance testee: speaks the protobuf conformance wire protocol on
+// stdin/stdout (4-byte LE length prefix + ConformanceRequest/Response), backed
+// by our reflective Protobuf lib. Run by the official conformance_test_runner.
+//
+//   PB_SRC=/path/to/protobuf bun conformance-testee.mjs
+//
+// We implement binary<->binary for proto3 + edition 2023. JSON / JSPB /
+// TEXT_FORMAT and proto2 message types are reported as `skipped`.
+import { readFileSync } from "node:fs";
+import { Schema } from "./serialization/protobuf/schema.ts";
+
+const P = process.env.PB_SRC;
+const read = (rel) => readFileSync(`${P}/${rel}`, "utf8");
+
+const conf = new Schema(read("conformance/conformance.proto"));
+const p3 = new Schema({ "google/protobuf/test_messages_proto3.proto": read("src/google/protobuf/test_messages_proto3.proto") });
+const ed = new Schema({ "x.proto": read("conformance/test_protos/test_messages_edition2023.proto") });
+const edP3 = new Schema({ "x.proto": read("editions/golden/test_messages_proto3_editions.proto") });
+const edP2 = new Schema({ "x.proto": read("editions/golden/test_messages_proto2_editions.proto") });
+
+const REQ = "conformance.ConformanceRequest";
+const RESP = "conformance.ConformanceResponse";
+
+function schemaFor(messageType) {
+  // Most-specific package prefix first.
+  if (messageType.startsWith("protobuf_test_messages.editions.proto3.")) return edP3;
+  if (messageType.startsWith("protobuf_test_messages.editions.proto2.")) return edP2;
+  if (messageType.startsWith("protobuf_test_messages.editions.")) return ed;
+  if (messageType.startsWith("protobuf_test_messages.proto3.")) return p3;
+  return null; // proto2 syntax (or unknown) — unsupported
+}
+
+function handle(reqBytes) {
+  const req = conf.parse(REQ, reqBytes);
+  const mt = req.messageType ?? "";
+
+  if (mt === "conformance.FailureSet") {
+    return conf.build(RESP, { protobufPayload: conf.build("conformance.FailureSet", {}) });
+  }
+
+  // We only do binary <-> binary.
+  const out = req.requestedOutputFormat; // enum name
+  if (req.jsonPayload !== undefined || req.jspbPayload !== undefined || req.textPayload !== undefined) {
+    return conf.build(RESP, { skipped: "non-protobuf input unsupported" });
+  }
+  if (out !== "PROTOBUF") {
+    return conf.build(RESP, { skipped: `output ${out} unsupported` });
+  }
+
+  const schema = schemaFor(mt);
+  if (!schema) return conf.build(RESP, { skipped: "proto2/unknown message type unsupported" });
+
+  try {
+    const msg = schema.parse(mt, req.protobufPayload ?? new Uint8Array(0));
+    try {
+      const bytes = schema.build(mt, msg);
+      return conf.build(RESP, { protobufPayload: bytes });
+    } catch (e) {
+      return conf.build(RESP, { serializeError: String(e?.message ?? e) });
+    }
+  } catch (e) {
+    const msg = String(e?.message ?? e);
+    if (msg.includes("unknown message")) return conf.build(RESP, { skipped: msg });
+    return conf.build(RESP, { parseError: msg });
+  }
+}
+
+// --- framed stdin/stdout loop ---
+const reader = Bun.stdin.stream().getReader();
+const sink = Bun.stdout.writer();
+let buf = new Uint8Array(0);
+
+function concat(a, b) {
+  const out = new Uint8Array(a.length + b.length);
+  out.set(a); out.set(b, a.length);
+  return out;
+}
+async function readExact(n) {
+  while (buf.length < n) {
+    const { done, value } = await reader.read();
+    if (done) return null;
+    buf = concat(buf, value);
+  }
+  const out = buf.subarray(0, n);
+  buf = buf.slice(n);
+  return out;
+}
+
+for (;;) {
+  const lenBuf = await readExact(4);
+  if (!lenBuf) break;
+  const len = new DataView(lenBuf.buffer, lenBuf.byteOffset, 4).getUint32(0, true);
+  const reqBytes = await readExact(len);
+  if (!reqBytes) break;
+
+  let resp;
+  try {
+    resp = handle(reqBytes.slice());
+  } catch (e) {
+    resp = conf.build(RESP, { runtimeError: String(e?.message ?? e) });
+  }
+
+  const header = new Uint8Array(4);
+  new DataView(header.buffer).setUint32(0, resp.length, true);
+  sink.write(header);
+  sink.write(resp);
+  sink.flush();
+}

crates/runtime/js/serialization/protobuf/decode.ts

@@ -50,13 +50,14 @@ function readSingle(r: Reader, type: FieldType): unknown {
   return decode(type.message, r.fork());
 }
 
-export function decode(message: MessageType, r: Reader): Record<string, unknown> {
-  const out: Record<string, unknown> = {};
+export function decode(message: MessageType, r: Reader, target?: Record<string, unknown>): Record<string, unknown> {
+  const out: Record<string, unknown> = target ?? {};
   while (!r.eof()) {
     const tagStart = r.pos;
     const tag = r.uint32();
     const fieldNo = tag >>> 3;
     const wire = tag & 7;
+    if (fieldNo === 0) throw new Error("protobuf: invalid field number 0");
     const field: Field | undefined = message.fieldByNumber.get(fieldNo);
 
     if (!field) {
@@ -107,14 +108,20 @@ export function decode(message: MessageType, r: Reader): Record<string, unknown>
       );
       continue;
     }
-    const value = readSingle(r, field.type);
     if (field.oneofIndex >= 0) {
       for (const num of message.oneofs[field.oneofIndex]!.fieldNumbers) {
         const other = message.fieldByNumber.get(num)!;
         if (other.jsonName !== field.jsonName) delete out[other.jsonName];
       }
     }
-    out[field.jsonName] = value;
+    if (field.type.kind === "message") {
+      // Repeated occurrences of a singular message field merge (proto spec).
+      const existing = out[field.jsonName];
+      const into = existing && typeof existing === "object" ? (existing as Record<string, unknown>) : undefined;
+      out[field.jsonName] = decode(field.type.message, r.fork(), into);
+    } else {
+      out[field.jsonName] = readSingle(r, field.type);
+    }
   }
   return out;
 }
@@ -130,5 +137,5 @@ function defaultForType(type: FieldType): unknown {
     }
   }
   if (type.kind === "enum") return type.enum.byNumber.get(0) ?? 0;
-  return undefined;
+  return {}; // message: default to an empty message (e.g. map<…, Message> missing value)
 }

crates/runtime/js/serialization/protobuf/parser.ts

@@ -169,8 +169,9 @@ class Parser {
           this.skipBlock();
           break;
         case "extend":
-          this.err("proto2 extensions (extend) are unsupported", t.line);
-        // eslint-disable-next-line no-fallthrough
+          this.lx.next();
+          this.skipBlock();
+          break;
         default:
           this.err(`unexpected '${t.value}' at top level`, t.line);
       }
@@ -231,8 +232,15 @@ class Parser {
             this.err("proto2 groups are unsupported", t.line);
           // eslint-disable-next-line no-fallthrough
           case "extensions":
+            // extension *range* declaration (valid in editions too) — ignore.
+            this.lx.next();
+            this.skipToSemicolon();
+            continue;
           case "extend":
-            this.err("proto2 extensions are unsupported", t.line);
+            // extension field definitions — skipped; such fields decode as unknown.
+            this.lx.next();
+            this.skipBlock();
+            continue;
         }
       }
       // otherwise a field
@@ -340,6 +348,11 @@ class Parser {
     const t = this.lx.next();
     if (t.kind === "str") return t.value;
     if (t.kind === "ident" || t.kind === "num") return t.value;
+    if (t.kind === "sym" && (t.value === "-" || t.value === "+")) {
+      const num = this.lx.next();
+      if (num.kind !== "num") this.err(`bad option value '${t.value}${num.value}'`, t.line);
+      return (t.value === "-" ? "-" : "") + num.value;
+    }
     if (t.kind === "sym" && t.value === "{") {
       // aggregate option value — skip to matching brace
       let depth = 1;
@@ -408,13 +421,19 @@ class Parser {
   }
 
   private parseInt32(): number {
+    let sign = 1;
+    const s = this.lx.peek();
+    if (s.kind === "sym" && (s.value === "-" || s.value === "+")) {
+      this.lx.next();
+      sign = s.value === "-" ? -1 : 1;
+    }
     const t = this.lx.next();
     if (t.kind !== "num") this.err(`expected number, got '${t.value}'`, t.line);
     const n = t.value.startsWith("0x") || t.value.startsWith("0X")
       ? parseInt(t.value, 16)
       : parseInt(t.value, 10);
     if (!Number.isFinite(n)) this.err(`bad number '${t.value}'`, t.line);
-    return n;
+    return sign * n;
   }
 
   /** Reads a (possibly dotted, possibly leading-dot) qualified name. */