Skip to content

Commit 30b0f40

Browse files
perf(parsers): bucket protobuf fields in a flat Vec, not a per-message HashMap
pb_bucket collected each message's wire fields into a HashMap<u32, Vec<WireVal>> — a heap allocation (and hashing) per decoded message, i.e. one per element in a large repeated field. Replace it with a flat Vec<(u32, WireVal)> scanned per descriptor field; singular fields (the common case) now allocate nothing for bucketing, and only present repeated/map fields gather a refs Vec. protobuf parse, 50k-element catalog: ~4.6s -> ~3.9s; protobuf_small ~78ms -> ~62ms (gap to node 4.0x -> 3.4x). Differential parity test still green; 80/80.
1 parent 295063e commit 30b0f40

1 file changed

Lines changed: 58 additions & 32 deletions

File tree

crates/runtime/src/parsers_ops.rs

Lines changed: 58 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -175,13 +175,14 @@ enum WireVal {
175175
Len(usize, usize),
176176
}
177177

178-
/// One pass over a message's wire bytes, bucketing values by field number.
179-
/// Repeated fields accumulate; singular fields keep every occurrence (the
180-
/// caller takes the last, per proto3 last-wins).
181-
fn pb_bucket(bytes: &[u8]) -> Result<std::collections::HashMap<u32, Vec<WireVal>>, String> {
178+
/// One pass over a message's wire bytes, collecting `(field number, value)` in
179+
/// wire order. A flat `Vec` (not a per-message `HashMap`) keeps the common case
180+
/// — small messages with each field once — allocation-light; the emit side scans
181+
/// it per descriptor field (field counts are small, so this stays cheap).
182+
fn pb_bucket(bytes: &[u8]) -> Result<Vec<(u32, WireVal)>, String> {
182183
use prost::bytes::Buf;
183184
use prost::encoding::{WireType, decode_key, decode_varint};
184-
let mut map: std::collections::HashMap<u32, Vec<WireVal>> = std::collections::HashMap::new();
185+
let mut entries: Vec<(u32, WireVal)> = Vec::new();
185186
let total = bytes.len();
186187
let mut buf: &[u8] = bytes;
187188
while !buf.is_empty() {
@@ -215,9 +216,18 @@ fn pb_bucket(bytes: &[u8]) -> Result<std::collections::HashMap<u32, Vec<WireVal>
215216
return Err("group wire types are not supported".into());
216217
}
217218
};
218-
map.entry(tag).or_default().push(v);
219+
entries.push((tag, v));
219220
}
220-
Ok(map)
221+
Ok(entries)
222+
}
223+
224+
/// The last value bucketed for `tag` (proto3 last-wins for singular fields).
225+
fn pb_last_by_tag(entries: &[(u32, WireVal)], tag: u32) -> Option<&WireVal> {
226+
entries
227+
.iter()
228+
.rev()
229+
.find(|(t, _)| *t == tag)
230+
.map(|(_, v)| v)
221231
}
222232

223233
/// Transcodes a length-delimited protobuf message into proto3 JSON, appended to
@@ -227,29 +237,45 @@ fn pb_transcode(
227237
bytes: &[u8],
228238
out: &mut String,
229239
) -> Result<(), String> {
230-
let buckets = pb_bucket(bytes)?;
240+
let entries = pb_bucket(bytes)?;
231241
out.push('{');
232242
let mut first = true;
233243
for field in desc.fields() {
234-
let Some(vals) = buckets.get(&field.number()) else {
235-
continue;
236-
};
237-
if vals.is_empty() {
238-
continue;
239-
}
240-
if !first {
241-
out.push(',');
242-
}
243-
first = false;
244-
pb_push_json_string(out, field.json_name());
245-
out.push(':');
246-
if field.is_map() {
247-
pb_emit_map(&field, vals, bytes, out)?;
248-
} else if field.is_list() {
249-
pb_emit_list(&field.kind(), vals, bytes, out)?;
244+
let num = field.number();
245+
if field.is_map() || field.is_list() {
246+
// Repeated/map fields may be scattered across the wire — gather every
247+
// occurrence in order. (Only allocates for fields actually present.)
248+
let vals: Vec<&WireVal> = entries
249+
.iter()
250+
.filter(|(t, _)| *t == num)
251+
.map(|(_, v)| v)
252+
.collect();
253+
if vals.is_empty() {
254+
continue;
255+
}
256+
if !first {
257+
out.push(',');
258+
}
259+
first = false;
260+
pb_push_json_string(out, field.json_name());
261+
out.push(':');
262+
if field.is_map() {
263+
pb_emit_map(&field, &vals, bytes, out)?;
264+
} else {
265+
pb_emit_list(&field.kind(), &vals, bytes, out)?;
266+
}
250267
} else {
251-
// Singular: proto3 last-wins.
252-
pb_emit_scalar(&field.kind(), vals.last().unwrap(), bytes, out)?;
268+
// Singular: proto3 last-wins; no allocation.
269+
let Some(v) = pb_last_by_tag(&entries, num) else {
270+
continue;
271+
};
272+
if !first {
273+
out.push(',');
274+
}
275+
first = false;
276+
pb_push_json_string(out, field.json_name());
277+
out.push(':');
278+
pb_emit_scalar(&field.kind(), v, bytes, out)?;
253279
}
254280
}
255281
out.push('}');
@@ -265,13 +291,13 @@ fn pb_is_packable(kind: &prost_reflect::Kind) -> bool {
265291

266292
fn pb_emit_list(
267293
kind: &prost_reflect::Kind,
268-
vals: &[WireVal],
294+
vals: &[&WireVal],
269295
bytes: &[u8],
270296
out: &mut String,
271297
) -> Result<(), String> {
272298
out.push('[');
273299
let mut first = true;
274-
for v in vals {
300+
for &v in vals {
275301
// A length-delimited value for a packable kind is a packed run of values.
276302
if let (WireVal::Len(s, l), true) = (v, pb_is_packable(kind)) {
277303
let mut pb: &[u8] = &bytes[*s..*s + *l];
@@ -396,7 +422,7 @@ fn pb_emit_scalar(
396422

397423
fn pb_emit_map(
398424
field: &prost_reflect::FieldDescriptor,
399-
vals: &[WireVal],
425+
vals: &[&WireVal],
400426
bytes: &[u8],
401427
out: &mut String,
402428
) -> Result<(), String> {
@@ -409,7 +435,7 @@ fn pb_emit_map(
409435
let value_field = entry.map_entry_value_field();
410436
out.push('{');
411437
let mut first = true;
412-
for v in vals {
438+
for &v in vals {
413439
let WireVal::Len(s, l) = v else {
414440
return Err("map entry is not length-delimited".into());
415441
};
@@ -421,15 +447,15 @@ fn pb_emit_map(
421447
first = false;
422448
// JSON object keys are always strings; render the key into one.
423449
let mut key = String::new();
424-
match eb.get(&key_field.number()).and_then(|x| x.last()) {
450+
match pb_last_by_tag(&eb, key_field.number()) {
425451
Some(kv) => pb_emit_scalar(&key_field.kind(), kv, entry_bytes, &mut key)?,
426452
None => pb_emit_default(&key_field.kind(), &mut key)?,
427453
}
428454
// Strip surrounding quotes if the key kind already rendered a JSON string.
429455
let key = key.trim_matches('"');
430456
pb_push_json_string(out, key);
431457
out.push(':');
432-
match eb.get(&value_field.number()).and_then(|x| x.last()) {
458+
match pb_last_by_tag(&eb, value_field.number()) {
433459
Some(vv) => pb_emit_scalar(&value_field.kind(), vv, entry_bytes, out)?,
434460
None => pb_emit_default_json(&value_field.kind(), out)?,
435461
}

0 commit comments

Comments
 (0)