From f06417faa2de87a864fadb86dd949a81931f94bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sat, 11 Jul 2026 22:27:56 +0200 Subject: [PATCH 01/24] AVRO-4299: [perl] Validate available bytes before allocating for length-prefixed values and collections A bytes or string value is a length prefix followed by that many bytes, and an array or map block is an element count followed by that many items. A malicious or truncated input can declare a huge length or count with little or no data. - _bytes_remaining reports the bytes still readable, found by seeking to the end and restoring the position; readers that cannot seek to the end (e.g. a streaming decompressor) yield undef and are simply skipped. - decode_bytes rejects an over-large declared length above a threshold before allocating. - decode_array/decode_map reject a block whose element count could not be backed by the bytes remaining, using _min_bytes_per_element computed from the element schema so a zero-byte element type (e.g. null) is not falsely rejected. Mirrors the Java SDK's checks (AVRO-4241). Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/perl/lib/Avro/BinaryDecoder.pm | 90 ++++++++++++++++++++++++++ lang/perl/t/03_bin_decode.t | 97 +++++++++++++++++++++++++++++ 2 files changed, 187 insertions(+) diff --git a/lang/perl/lib/Avro/BinaryDecoder.pm b/lang/perl/lib/Avro/BinaryDecoder.pm index d2bb65e14f7..66c39dce7ba 100644 --- a/lang/perl/lib/Avro/BinaryDecoder.pm +++ b/lang/perl/lib/Avro/BinaryDecoder.pm @@ -132,10 +132,95 @@ sub decode_bytes { my $class = shift; my $reader = pop; my $size = decode_long($class, undef, undef, $reader); + _ensure_available($reader, $size); $reader->read(my $buf, $size); return $buf; } +## Reject a declared length that exceeds the bytes actually remaining before +## allocating for it, to guard against an out-of-memory attack from a malicious +## or truncated input. Only enforced for larger reads and when the reader can +## report its size via seek/tell; smaller reads and non-seekable readers fall +## through to a direct read. +use constant _MAX_UNCHECKED_READ => 1024 * 1024; + +## Number of bytes still available to read, or undef when the reader cannot +## report its size. Used to reject a declared length or collection block count +## that exceeds the data actually available before allocating for it. Readers +## that do not support seeking to the end (e.g. a streaming decompressor) return +## undef, so the check is simply skipped for them. +sub _bytes_remaining { + my ($reader) = @_; + my $current = eval { $reader->tell }; + return undef if !defined $current || $current < 0; + return eval { + $reader->seek(0, 2) # SEEK_END + or die "seek to end failed\n"; + my $end = $reader->tell; + $reader->seek($current, 0); # SEEK_SET, restore position + die "unknown end position\n" if !defined $end || $end < 0; + $end - $current; + }; +} + +sub _ensure_available { + my ($reader, $size) = @_; + return if $size <= _MAX_UNCHECKED_READ; + my $remaining = _bytes_remaining($reader); + return unless defined $remaining; + if ($size > $remaining) { + throw Avro::Schema::Error::Parse( + "Cannot read $size bytes, only $remaining remaining"); + } + return; +} + +## Minimum number of bytes a single value of the given schema can occupy on the +## wire. Used to reject an array/map block count that could not be backed by the +## bytes remaining. A type that can encode to zero bytes (null) returns 0, which +## disables the collection check for it (so an array of nulls is not falsely +## rejected). +sub _min_bytes_per_element { + my ($schema, $visited) = @_; + $visited ||= {}; + my $type = $schema->type; + return 0 if $type eq 'null'; + return 4 if $type eq 'float'; + return 8 if $type eq 'double'; + return $schema->size if $type eq 'fixed'; + if ($type eq 'record' || $type eq 'error') { + my $id = "$schema"; # stringified reference as identity + return 0 if $visited->{$id}; + $visited->{$id} = 1; + my $total = 0; + for my $field (@{ $schema->fields }) { + $total += _min_bytes_per_element($field->{type}, $visited); + } + delete $visited->{$id}; + return $total; + } + # boolean, int, long, bytes, string, enum, union, array, map: >= 1 byte + # (a union encodes at least a 1-byte branch index). + return 1; +} + +## Reject a collection (array or map) block whose declared element count could +## not be backed by the bytes actually remaining, before iterating. Skipped when +## the per-element minimum is zero or when the reader cannot report how many +## bytes remain. +sub _ensure_collection_available { + my ($reader, $count, $min_bytes) = @_; + return if $count <= 0 || $min_bytes <= 0; + my $remaining = _bytes_remaining($reader); + return unless defined $remaining; + if ($count * $min_bytes > $remaining) { + throw Avro::Schema::Error::Parse( + "Collection claims $count elements with at least $min_bytes bytes each, " + . "but only $remaining bytes are available"); + } + return; +} + sub skip_string { &skip_bytes } sub decode_string { my $class = shift; @@ -251,6 +336,7 @@ sub decode_array { my @array; my $writer_items = $writer_schema->items; my $reader_items = $reader_schema->items; + my $min_bytes = _min_bytes_per_element($writer_items); while ($block_count) { my $block_size; if ($block_count < 0) { @@ -258,6 +344,7 @@ sub decode_array { $block_size = decode_long($class, @_); ## XXX we can skip with $reader_schema? } + _ensure_collection_available($reader, $block_count, $min_bytes); for (1..$block_count) { push @array, $class->decode( writer_schema => $writer_items, @@ -296,6 +383,8 @@ sub decode_map { my $block_count = decode_long($class, @_); my $writer_values = $writer_schema->values; my $reader_values = $reader_schema->values; + # Map keys are strings (>= 1 byte length prefix) plus the value. + my $min_bytes = 1 + _min_bytes_per_element($writer_values); while ($block_count) { my $block_size; if ($block_count < 0) { @@ -303,6 +392,7 @@ sub decode_map { $block_size = decode_long($class, @_); ## XXX we can skip with $reader_schema? } + _ensure_collection_available($reader, $block_count, $min_bytes); for (1..$block_count) { my $key = decode_string($class, @_); unless (defined $key && length $key) { diff --git a/lang/perl/t/03_bin_decode.t b/lang/perl/t/03_bin_decode.t index e5da35d9983..65906e4d7aa 100644 --- a/lang/perl/t/03_bin_decode.t +++ b/lang/perl/t/03_bin_decode.t @@ -255,4 +255,101 @@ EOP is $dec->{one}[0], 1.0, "kind of dumb test"; } +## A bytes/string value declares a length prefix; a malicious or truncated +## input can declare far more bytes than actually exist. On a seekable reader +## that is rejected before allocating for it. +{ + my $bytes_schema = Avro::Schema->parse(q({ "type": "bytes" })); + + ## A length prefix declaring 100 MiB, with no payload following it. + my $enc = ''; + Avro::BinaryEncoder->encode( + schema => Avro::Schema->parse(q({ "type": "long" })), + data => 100 * 1024 * 1024, + emit_cb => sub { $enc .= ${ $_[0] } }, + ); + open my $reader, '<', \$enc or die "Can't open memory file: $!"; + throws_ok { + Avro::BinaryDecoder->decode( + writer_schema => $bytes_schema, + reader_schema => $bytes_schema, + reader => $reader, + ); + } qr/Cannot read/, "oversized bytes length is rejected before allocating"; + + ## A well-formed value above the check threshold whose data is present + ## still decodes. + my $payload = 'x' x (2 * 1024 * 1024); + my $enc2 = ''; + Avro::BinaryEncoder->encode( + schema => $bytes_schema, + data => $payload, + emit_cb => sub { $enc2 .= ${ $_[0] } }, + ); + open my $reader2, '<', \$enc2 or die "Can't open memory file: $!"; + my $dec = Avro::BinaryDecoder->decode( + writer_schema => $bytes_schema, + reader_schema => $bytes_schema, + reader => $reader2, + ); + is $dec, $payload, "within-limit bytes value still decodes"; +} + +## An array/map block declares an element count; a malicious or truncated input +## can declare far more elements than the remaining bytes could hold. The count +## is validated against the bytes remaining before iterating, using the minimum +## on-wire size of the element schema (so 0-byte elements like null are not +## falsely rejected). +sub encode_long { + my ($value) = @_; + my $enc = ''; + Avro::BinaryEncoder->encode( + schema => Avro::Schema->parse(q({ "type": "long" })), + data => $value, + emit_cb => sub { $enc .= ${ $_[0] } }, + ); + return $enc; +} + +{ + my $array_schema = Avro::Schema->parse(q({ "type": "array", "items": "long" })); + my $enc = encode_long(1_000_000); # 1,000,000 longs, no element data + open my $reader, '<', \$enc or die "Can't open memory file: $!"; + throws_ok { + Avro::BinaryDecoder->decode( + writer_schema => $array_schema, + reader_schema => $array_schema, + reader => $reader, + ); + } qr/Collection claims/, "oversized array count is rejected before iterating"; +} + +{ + my $map_schema = Avro::Schema->parse(q({ "type": "map", "values": "long" })); + my $enc = encode_long(1_000_000); + open my $reader, '<', \$enc or die "Can't open memory file: $!"; + throws_ok { + Avro::BinaryDecoder->decode( + writer_schema => $map_schema, + reader_schema => $map_schema, + reader => $reader, + ); + } qr/Collection claims/, "oversized map count is rejected before iterating"; +} + +{ + ## An array of nulls: null elements occupy zero bytes, so a large count is + ## legitimate and must not be rejected. + my $array_schema = Avro::Schema->parse(q({ "type": "array", "items": "null" })); + my $count = 100_000; + my $enc = encode_long($count) . encode_long(0); # one block + end marker + open my $reader, '<', \$enc or die "Can't open memory file: $!"; + my $dec = Avro::BinaryDecoder->decode( + writer_schema => $array_schema, + reader_schema => $array_schema, + reader => $reader, + ); + is scalar(@$dec), $count, "array of nulls is not falsely rejected"; +} + done_testing; From e51a7b1f5af0844379ac0f20c27d97f7ae04811e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sat, 11 Jul 2026 23:46:27 +0200 Subject: [PATCH 02/24] AVRO-4299: [perl] Avoid explicit return undef in _bytes_remaining The Lint CI step (perlcritic, default severity) flagged a Subroutines::ProhibitExplicitReturnUndef violation: 'return undef' in _bytes_remaining. Use a bare 'return', which yields undef in the scalar context the callers use, so a reader that cannot report its size still skips the check. Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/perl/lib/Avro/BinaryDecoder.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lang/perl/lib/Avro/BinaryDecoder.pm b/lang/perl/lib/Avro/BinaryDecoder.pm index 66c39dce7ba..5b725d6d91a 100644 --- a/lang/perl/lib/Avro/BinaryDecoder.pm +++ b/lang/perl/lib/Avro/BinaryDecoder.pm @@ -152,7 +152,7 @@ use constant _MAX_UNCHECKED_READ => 1024 * 1024; sub _bytes_remaining { my ($reader) = @_; my $current = eval { $reader->tell }; - return undef if !defined $current || $current < 0; + return if !defined $current || $current < 0; return eval { $reader->seek(0, 2) # SEEK_END or die "seek to end failed\n"; From 92df248905f6ee2f755d87493c27e2e805e8cb88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sun, 12 Jul 2026 00:27:58 +0200 Subject: [PATCH 03/24] AVRO-4299: [perl] Reject negative lengths; restore position robustly; tighter map key min Review feedback: - decode_bytes rejects a negative decoded length before reading, since Avro bytes/string lengths must be non-negative. - _bytes_remaining now always restores the original position after seeking to the end (even if reading the end offset fails), and treats a restore failure as fatal, so the reader is never left at EOF and silently used for subsequent decoding. - decode_map keys are non-empty strings (empty keys are rejected), so the minimum per-entry size uses 2 bytes for the key (1-byte length prefix plus at least 1 byte of data) instead of 1, making the pre-check slightly tighter. Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/perl/lib/Avro/BinaryDecoder.pm | 34 ++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/lang/perl/lib/Avro/BinaryDecoder.pm b/lang/perl/lib/Avro/BinaryDecoder.pm index 5b725d6d91a..ba7c56637b2 100644 --- a/lang/perl/lib/Avro/BinaryDecoder.pm +++ b/lang/perl/lib/Avro/BinaryDecoder.pm @@ -132,6 +132,10 @@ sub decode_bytes { my $class = shift; my $reader = pop; my $size = decode_long($class, undef, undef, $reader); + if ($size < 0) { + throw Avro::Schema::Error::Parse( + "Invalid negative bytes/string length: $size"); + } _ensure_available($reader, $size); $reader->read(my $buf, $size); return $buf; @@ -153,14 +157,22 @@ sub _bytes_remaining { my ($reader) = @_; my $current = eval { $reader->tell }; return if !defined $current || $current < 0; - return eval { - $reader->seek(0, 2) # SEEK_END - or die "seek to end failed\n"; - my $end = $reader->tell; - $reader->seek($current, 0); # SEEK_SET, restore position - die "unknown end position\n" if !defined $end || $end < 0; - $end - $current; - }; + + # Attempt to seek to the end. Readers that cannot (e.g. a streaming + # decompressor) leave the position unchanged; skip the check for them. + my $moved = eval { $reader->seek(0, 2) }; # SEEK_END + return if !$moved; + + # We are now at the end. Record the end offset, then ALWAYS restore the + # original position. A restore failure leaves the reader corrupted for + # subsequent decoding, so treat it as fatal rather than continuing. + my $end = eval { $reader->tell }; + unless (eval { $reader->seek($current, 0) }) { # SEEK_SET + throw Avro::Schema::Error::Parse( + "Failed to restore reader position after size check"); + } + return if !defined $end || $end < 0; + return $end - $current; } sub _ensure_available { @@ -383,8 +395,10 @@ sub decode_map { my $block_count = decode_long($class, @_); my $writer_values = $writer_schema->values; my $reader_values = $reader_schema->values; - # Map keys are strings (>= 1 byte length prefix) plus the value. - my $min_bytes = 1 + _min_bytes_per_element($writer_values); + # A map key is a non-empty string (decode_map rejects empty keys below), so + # its minimum on-wire size is 2 bytes: a 1-byte length prefix plus at least + # 1 byte of key data, in addition to the value. + my $min_bytes = 2 + _min_bytes_per_element($writer_values); while ($block_count) { my $block_size; if ($block_count < 0) { From 98aaeb61e8d7a670baa229a272fbbbf5e33a49c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sun, 12 Jul 2026 01:05:59 +0200 Subject: [PATCH 04/24] AVRO-4299: [perl] Compare collection size via division to avoid overflow Review feedback: $count * $min_bytes could overflow/wrap on some Perl builds (notably 32-bit), defeating the pre-check for very large declared counts. Compare $count against int($remaining / $min_bytes) instead. Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/perl/lib/Avro/BinaryDecoder.pm | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lang/perl/lib/Avro/BinaryDecoder.pm b/lang/perl/lib/Avro/BinaryDecoder.pm index ba7c56637b2..6f61e703bbb 100644 --- a/lang/perl/lib/Avro/BinaryDecoder.pm +++ b/lang/perl/lib/Avro/BinaryDecoder.pm @@ -225,7 +225,9 @@ sub _ensure_collection_available { return if $count <= 0 || $min_bytes <= 0; my $remaining = _bytes_remaining($reader); return unless defined $remaining; - if ($count * $min_bytes > $remaining) { + # Compare via integer division rather than multiplying, so $count * $min_bytes + # cannot overflow/wrap (notably on 32-bit builds) and defeat the check. + if ($count > int($remaining / $min_bytes)) { throw Avro::Schema::Error::Parse( "Collection claims $count elements with at least $min_bytes bytes each, " . "but only $remaining bytes are available"); From fa8332b5f63b87f12eec2e8805e9642b7dc26d25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sun, 12 Jul 2026 01:30:17 +0200 Subject: [PATCH 05/24] AVRO-4299: [perl] Reject short reads and negative skip lengths Review feedback: - decode_bytes now verifies that read() returned exactly the declared number of bytes; a short read (truncated input) is rejected instead of silently returning an under-sized buffer that would misalign the decoder. - skip_bytes (used by skip_string during schema resolution) rejects a negative declared length, consistent with decode_bytes. Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/perl/lib/Avro/BinaryDecoder.pm | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lang/perl/lib/Avro/BinaryDecoder.pm b/lang/perl/lib/Avro/BinaryDecoder.pm index 6f61e703bbb..d538a0f5ffc 100644 --- a/lang/perl/lib/Avro/BinaryDecoder.pm +++ b/lang/perl/lib/Avro/BinaryDecoder.pm @@ -124,6 +124,10 @@ sub skip_bytes { my $class = shift; my $reader = pop; my $size = decode_long($class, undef, undef, $reader); + if ($size < 0) { + throw Avro::Schema::Error::Parse( + "Invalid negative bytes/string length: $size"); + } $reader->seek($size, 0); return; } @@ -137,7 +141,13 @@ sub decode_bytes { "Invalid negative bytes/string length: $size"); } _ensure_available($reader, $size); - $reader->read(my $buf, $size); + my $nread = $reader->read(my $buf, $size); + if (!defined $nread || $nread != $size) { + # A short read means the input is truncated; failing here avoids + # returning an under-sized buffer and misaligning the decoder. + throw Avro::Schema::Error::Parse( + "Expected $size bytes but read " . (defined $nread ? $nread : 0)); + } return $buf; } From 9a4cf0a26b882560811928010f837478a9619b52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sun, 12 Jul 2026 07:25:11 +0200 Subject: [PATCH 06/24] AVRO-4299: [perl] Skip bytes relative to the current position Review feedback: skip_bytes() used seek($size, 0), i.e. SEEK_SET, which seeks to the absolute offset $size instead of skipping $size bytes forward from the current position. Use whence 1 (SEEK_CUR) for a relative skip. Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/perl/lib/Avro/BinaryDecoder.pm | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lang/perl/lib/Avro/BinaryDecoder.pm b/lang/perl/lib/Avro/BinaryDecoder.pm index d538a0f5ffc..1a247b92525 100644 --- a/lang/perl/lib/Avro/BinaryDecoder.pm +++ b/lang/perl/lib/Avro/BinaryDecoder.pm @@ -128,7 +128,10 @@ sub skip_bytes { throw Avro::Schema::Error::Parse( "Invalid negative bytes/string length: $size"); } - $reader->seek($size, 0); + # Skip forward by $size bytes relative to the current position + # (SEEK_CUR); whence 0 (SEEK_SET) would incorrectly seek to the + # absolute offset $size. + $reader->seek($size, 1); return; } From 25ab8ab2667ddbe56efa1a0ad1345f67d8c1a2b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sun, 12 Jul 2026 08:05:12 +0200 Subject: [PATCH 07/24] AVRO-4299: [perl] Validate and relatively-skip bytes/fixed/block fields Review feedback: skip_bytes could silently succeed when the declared length exceeded the bytes actually available (a seek past EOF is allowed on many handles), causing subsequent decoding to read EOF as zeros instead of failing on truncated input. - skip_bytes now calls _ensure_available before skipping (mirroring decode_bytes) and treats a failed seek as fatal. - skip_fixed used seek(size, 0) (SEEK_SET, absolute); use SEEK_CUR for a relative skip and throw on a failed seek. - skip_block's negative-count branch ignored the block-size long, did an absolute seek of a negative offset, and re-looped on the same count (an infinite loop). Read the block size and skip it relatively (SEEK_CUR), throwing on a failed seek. Added tests covering skipping fixed/bytes writer fields absent from the reader schema and rejecting an oversized skipped bytes length. Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/perl/lib/Avro/BinaryDecoder.pm | 35 ++++++++++++++---- lang/perl/t/03_bin_decode.t | 57 +++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 7 deletions(-) diff --git a/lang/perl/lib/Avro/BinaryDecoder.pm b/lang/perl/lib/Avro/BinaryDecoder.pm index 1a247b92525..9db606eb409 100644 --- a/lang/perl/lib/Avro/BinaryDecoder.pm +++ b/lang/perl/lib/Avro/BinaryDecoder.pm @@ -128,10 +128,17 @@ sub skip_bytes { throw Avro::Schema::Error::Parse( "Invalid negative bytes/string length: $size"); } - # Skip forward by $size bytes relative to the current position - # (SEEK_CUR); whence 0 (SEEK_SET) would incorrectly seek to the - # absolute offset $size. - $reader->seek($size, 1); + # Reject a declared length that exceeds the bytes actually remaining before + # skipping, mirroring decode_bytes: a seek past EOF silently succeeds on + # many handles, so without this a truncated input would later read zeros + # instead of failing. + _ensure_available($reader, $size); + # Skip forward by $size bytes relative to the current position (SEEK_CUR); + # whence 0 (SEEK_SET) would incorrectly seek to the absolute offset $size. + # A failed seek means the reader cannot skip and is treated as fatal. + unless ($reader->seek($size, 1)) { + throw Avro::Schema::Error::Parse("Failed to skip $size bytes"); + } return; } @@ -332,8 +339,16 @@ sub skip_block { my $block_count = decode_long($class, undef, undef, $reader); while ($block_count) { if ($block_count < 0) { - $reader->seek($block_count, 0); - next; + # A negative count is followed by a long block size in bytes, which + # lets the whole block be skipped without decoding each item. Skip + # forward by that many bytes relative to the current position + # (SEEK_CUR); whence 0 (SEEK_SET) would seek to an absolute (here + # nonsensical) offset. A failed seek is treated as fatal. + my $block_size = decode_long($class, undef, undef, $reader); + unless ($reader->seek($block_size, 1)) { + throw Avro::Schema::Error::Parse( + "Failed to skip block of $block_size bytes"); + } } else { for (1..$block_count) { @@ -469,7 +484,13 @@ sub decode_union { sub skip_fixed { my $class = shift; my ($schema, $reader) = @_; - $reader->seek($schema->size, 0); + # Skip the fixed-size payload relative to the current position (SEEK_CUR); + # whence 0 (SEEK_SET) would incorrectly seek to the absolute offset. A + # failed seek is treated as fatal. + unless ($reader->seek($schema->size, 1)) { + throw Avro::Schema::Error::Parse( + "Failed to skip " . $schema->size . " bytes"); + } } ## 1.3.2 Fixed instances are encoded using the number of bytes declared in the diff --git a/lang/perl/t/03_bin_decode.t b/lang/perl/t/03_bin_decode.t index 65906e4d7aa..510258f7c7d 100644 --- a/lang/perl/t/03_bin_decode.t +++ b/lang/perl/t/03_bin_decode.t @@ -352,4 +352,61 @@ sub encode_long { is scalar(@$dec), $count, "array of nulls is not falsely rejected"; } +## Skipping writer fields absent from the reader schema must advance the reader +## by the correct relative amount (SEEK_CUR), including fixed and bytes fields. +{ + my $w_schema = Avro::Schema->parse(<<'EOJ'); + { "type": "record", "name": "test", + "fields" : [ + {"name": "f", "type": {"type": "fixed", "name": "fix8", "size": 8}}, + {"name": "b", "type": "bytes"}, + {"name": "a", "type": "long"} ]} +EOJ + my $r_schema = Avro::Schema->parse(<<'EOJ'); + { "type": "record", "name": "test", + "fields" : [ {"name": "a", "type": "long"} ]} +EOJ + my $data = { f => "01234567", b => "payload", a => 42 }; + my $enc = ''; + Avro::BinaryEncoder->encode( + schema => $w_schema, + data => $data, + emit_cb => sub { $enc .= ${ $_[0] } }, + ); + open my $reader, '<', \$enc or die "Cannot open memory file: $!"; + my $dec = Avro::BinaryDecoder->decode( + writer_schema => $w_schema, + reader_schema => $r_schema, + reader => $reader, + ); + is $dec->{a}, 42, "long read correctly after skipping fixed and bytes fields"; + ok ! exists $dec->{f}, "skipped fixed field absent"; + ok ! exists $dec->{b}, "skipped bytes field absent"; +} + +## A skipped bytes field whose declared length far exceeds the remaining data +## must be rejected rather than silently seeking past EOF. +{ + my $w_schema = Avro::Schema->parse(<<'EOJ'); + { "type": "record", "name": "test", + "fields" : [ + {"name": "b", "type": "bytes"}, + {"name": "a", "type": "long"} ]} +EOJ + my $r_schema = Avro::Schema->parse(<<'EOJ'); + { "type": "record", "name": "test", + "fields" : [ {"name": "a", "type": "long"} ]} +EOJ + # Declare a bytes length of 100 MiB but provide no data for it. + my $enc = encode_long(100 * 1024 * 1024); + open my $reader, '<', \$enc or die "Cannot open memory file: $!"; + throws_ok { + Avro::BinaryDecoder->decode( + writer_schema => $w_schema, + reader_schema => $r_schema, + reader => $reader, + ); + } qr/Cannot read/, "oversized skipped bytes length is rejected"; +} + done_testing; From 8af98bef577d29ecea0cc2234039a788a6a5b296 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sun, 12 Jul 2026 09:00:53 +0200 Subject: [PATCH 08/24] AVRO-4299: [perl] Reject a negative block size in skip_block Review feedback: in skip_block's negative-count path the decoded block-size long was not validated. A corrupted input with a negative block size would make seek($block_size, SEEK_CUR) move the reader backwards, risking an infinite loop or mis-decoding. Reject a negative block size, and validate it against the bytes remaining (via _ensure_available) before seeking. Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/perl/lib/Avro/BinaryDecoder.pm | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lang/perl/lib/Avro/BinaryDecoder.pm b/lang/perl/lib/Avro/BinaryDecoder.pm index 9db606eb409..6c8b8c2f988 100644 --- a/lang/perl/lib/Avro/BinaryDecoder.pm +++ b/lang/perl/lib/Avro/BinaryDecoder.pm @@ -345,6 +345,16 @@ sub skip_block { # (SEEK_CUR); whence 0 (SEEK_SET) would seek to an absolute (here # nonsensical) offset. A failed seek is treated as fatal. my $block_size = decode_long($class, undef, undef, $reader); + if ($block_size < 0) { + # A negative block size would seek the reader backwards, + # risking an infinite loop or mis-decoding of a corrupt input. + throw Avro::Schema::Error::Parse( + "Invalid negative block size: $block_size"); + } + # Reject a block size that exceeds the bytes actually remaining + # before skipping, so a truncated input fails instead of seeking + # past EOF. + _ensure_available($reader, $block_size); unless ($reader->seek($block_size, 1)) { throw Avro::Schema::Error::Parse( "Failed to skip block of $block_size bytes"); From fce9c6a85e429f794760585295026a36644e067c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sun, 12 Jul 2026 09:36:58 +0200 Subject: [PATCH 09/24] AVRO-4299: [perl] Fix skip_block call signature for array/map skipping Review feedback: skip_block used a method-style signature (my $class = shift) but its only callers, skip_array and skip_map, invoke it as a plain function skip_block($reader, $coderef). That made $class capture the reader handle and $reader capture the coderef, so decode_long would try $coderef->read and die whenever an array or map field was skipped during schema projection. Drop the implicit $class parameter so the plain-function call style works, and pass __PACKAGE__ to decode_long (which ignores it). Added a test that skips writer array and map fields absent from the reader schema. Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/perl/lib/Avro/BinaryDecoder.pm | 11 ++++++---- lang/perl/t/03_bin_decode.t | 33 +++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/lang/perl/lib/Avro/BinaryDecoder.pm b/lang/perl/lib/Avro/BinaryDecoder.pm index 6c8b8c2f988..20c1d6052ba 100644 --- a/lang/perl/lib/Avro/BinaryDecoder.pm +++ b/lang/perl/lib/Avro/BinaryDecoder.pm @@ -334,9 +334,12 @@ sub decode_enum { } sub skip_block { - my $class = shift; + # Called as a plain function by skip_array/skip_map as + # skip_block($reader, $coderef); it is not a method, so there is no leading + # class argument. decode_long ignores its (shifted) first argument and reads + # from the last, so pass __PACKAGE__ for it and $reader last. my ($reader, $block_content) = @_; - my $block_count = decode_long($class, undef, undef, $reader); + my $block_count = decode_long(__PACKAGE__, undef, undef, $reader); while ($block_count) { if ($block_count < 0) { # A negative count is followed by a long block size in bytes, which @@ -344,7 +347,7 @@ sub skip_block { # forward by that many bytes relative to the current position # (SEEK_CUR); whence 0 (SEEK_SET) would seek to an absolute (here # nonsensical) offset. A failed seek is treated as fatal. - my $block_size = decode_long($class, undef, undef, $reader); + my $block_size = decode_long(__PACKAGE__, undef, undef, $reader); if ($block_size < 0) { # A negative block size would seek the reader backwards, # risking an infinite loop or mis-decoding of a corrupt input. @@ -365,7 +368,7 @@ sub skip_block { $block_content->(); } } - $block_count = decode_long($class, undef, undef, $reader); + $block_count = decode_long(__PACKAGE__, undef, undef, $reader); } } diff --git a/lang/perl/t/03_bin_decode.t b/lang/perl/t/03_bin_decode.t index 510258f7c7d..2c642e3edc3 100644 --- a/lang/perl/t/03_bin_decode.t +++ b/lang/perl/t/03_bin_decode.t @@ -384,6 +384,39 @@ EOJ ok ! exists $dec->{b}, "skipped bytes field absent"; } +## Skipping writer array/map fields absent from the reader schema exercises +## skip_block, which is called as a plain function (not a method) by +## skip_array/skip_map. +{ + my $w_schema = Avro::Schema->parse(<<'EOJ'); + { "type": "record", "name": "test", + "fields" : [ + {"name": "arr", "type": {"type": "array", "items": "long"}}, + {"name": "map", "type": {"type": "map", "values": "string"}}, + {"name": "a", "type": "long"} ]} +EOJ + my $r_schema = Avro::Schema->parse(<<'EOJ'); + { "type": "record", "name": "test", + "fields" : [ {"name": "a", "type": "long"} ]} +EOJ + my $data = { arr => [ 1, 2, 3 ], map => { x => "y", z => "w" }, a => 99 }; + my $enc = ''; + Avro::BinaryEncoder->encode( + schema => $w_schema, + data => $data, + emit_cb => sub { $enc .= ${ $_[0] } }, + ); + open my $reader, '<', \$enc or die "Cannot open memory file: $!"; + my $dec = Avro::BinaryDecoder->decode( + writer_schema => $w_schema, + reader_schema => $r_schema, + reader => $reader, + ); + is $dec->{a}, 99, "long read correctly after skipping array and map fields"; + ok ! exists $dec->{arr}, "skipped array field absent"; + ok ! exists $dec->{map}, "skipped map field absent"; +} + ## A skipped bytes field whose declared length far exceeds the remaining data ## must be rejected rather than silently seeking past EOF. { From ddca5b5a6161b73bf815834e10fdd2076a358545 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sun, 12 Jul 2026 13:52:57 +0200 Subject: [PATCH 10/24] AVRO-4299: [perl] Cap zero-byte collection element allocation Completes the available-bytes protection for collections and supersedes the separate collection-limit change. Elements whose schema encodes to zero bytes (null, or a record with only zero-byte fields) consume no input, so the bytes-remaining check cannot bound their count. A tiny payload declaring a huge array block count of such elements drove an unbounded allocation. _ensure_collection_available now enforces the bytes-remaining check plus a zero-byte cap ($MAX_COLLECTION_ITEMS, default 10,000,000) and a structural cap on all collections ($MAX_COLLECTION_STRUCTURAL, default Integer.MAX_VALUE - 8) that also covers non-seekable readers. AVRO_MAX_COLLECTION_ITEMS, when set, caps both. Rejections raise the dedicated Avro::BinaryDecoder::Error::CollectionSize. skip_block is now element-aware (bounds zero-byte skips tightly, other skips by the structural cap) so a huge block cannot loop unboundedly. Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/perl/lib/Avro/BinaryDecoder.pm | 98 +++++++++++++---- lang/perl/t/03_bin_decode.t | 156 ++++++++++++++++++++++++++++ 2 files changed, 236 insertions(+), 18 deletions(-) diff --git a/lang/perl/lib/Avro/BinaryDecoder.pm b/lang/perl/lib/Avro/BinaryDecoder.pm index 20c1d6052ba..d6e9ff15741 100644 --- a/lang/perl/lib/Avro/BinaryDecoder.pm +++ b/lang/perl/lib/Avro/BinaryDecoder.pm @@ -168,6 +168,31 @@ sub decode_bytes { ## through to a direct read. use constant _MAX_UNCHECKED_READ => 1024 * 1024; +## Maximum number of zero-byte-encoded collection elements (e.g. an array of +## nulls) to allocate from a single decode. Such elements consume no input, so +## the bytes-remaining check cannot bound their count; without a cap a tiny +## payload can declare a huge block count and exhaust memory. Overridable via the +## AVRO_MAX_COLLECTION_ITEMS environment variable, or by setting +## $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS directly. +our $DEFAULT_MAX_COLLECTION_ITEMS = 10_000_000; +## Structural cap on the number of elements in any array or map (an +## overflow/defense-in-depth guard), matching the historical Integer.MAX_VALUE-8 +## limit. Non-zero-byte elements are also bounded by the bytes remaining. +our $DEFAULT_MAX_COLLECTION_STRUCTURAL = (2 ** 31) - 8; + +## AVRO_MAX_COLLECTION_ITEMS, when set, caps both limits; otherwise zero-byte +## elements use the tighter default and all collections use the structural cap. +## Both may also be overridden directly (e.g. in tests). +our $MAX_COLLECTION_ITEMS; +our $MAX_COLLECTION_STRUCTURAL; +if ( defined $ENV{AVRO_MAX_COLLECTION_ITEMS} && $ENV{AVRO_MAX_COLLECTION_ITEMS} =~ /\A[0-9]+\z/ ) { + $MAX_COLLECTION_ITEMS = $MAX_COLLECTION_STRUCTURAL = $ENV{AVRO_MAX_COLLECTION_ITEMS} + 0; +} +else { + $MAX_COLLECTION_ITEMS = $DEFAULT_MAX_COLLECTION_ITEMS; + $MAX_COLLECTION_STRUCTURAL = $DEFAULT_MAX_COLLECTION_STRUCTURAL; +} + ## Number of bytes still available to read, or undef when the reader cannot ## report its size. Used to reject a declared length or collection block count ## that exceeds the data actually available before allocating for it. Readers @@ -241,16 +266,32 @@ sub _min_bytes_per_element { ## the per-element minimum is zero or when the reader cannot report how many ## bytes remain. sub _ensure_collection_available { - my ($reader, $count, $min_bytes) = @_; - return if $count <= 0 || $min_bytes <= 0; - my $remaining = _bytes_remaining($reader); - return unless defined $remaining; - # Compare via integer division rather than multiplying, so $count * $min_bytes - # cannot overflow/wrap (notably on 32-bit builds) and defeat the check. - if ($count > int($remaining / $min_bytes)) { - throw Avro::Schema::Error::Parse( - "Collection claims $count elements with at least $min_bytes bytes each, " - . "but only $remaining bytes are available"); + my ($reader, $existing, $count, $min_bytes) = @_; + return if $count <= 0; + if ($min_bytes > 0) { + my $remaining = _bytes_remaining($reader); + # Compare via integer division rather than multiplying, so $count * $min_bytes + # cannot overflow/wrap (notably on 32-bit builds) and defeat the check. + if (defined $remaining && $count > int($remaining / $min_bytes)) { + throw Avro::Schema::Error::Parse( + "Collection claims $count elements with at least $min_bytes bytes each, " + . "but only $remaining bytes are available"); + } + # Structural / overflow guard, also covering non-seekable readers where + # the bytes check above cannot run. + if ($existing + $count > $MAX_COLLECTION_STRUCTURAL) { + throw Avro::BinaryDecoder::Error::CollectionSize( + "Cannot read a collection of more than $MAX_COLLECTION_STRUCTURAL " + . "elements (declared @{[ $existing + $count ]})"); + } + } + # Zero-byte elements (e.g. null) consume no input, so they cannot be bounded + # by the bytes remaining; cap their cumulative count instead. + elsif ($existing + $count > $MAX_COLLECTION_ITEMS) { + throw Avro::BinaryDecoder::Error::CollectionSize( + "Cannot read a collection of more than $MAX_COLLECTION_ITEMS zero-byte " + . "elements (declared @{[ $existing + $count ]}); raise " + . "\$Avro::BinaryDecoder::MAX_COLLECTION_ITEMS if this is legitimate"); } return; } @@ -335,11 +376,16 @@ sub decode_enum { sub skip_block { # Called as a plain function by skip_array/skip_map as - # skip_block($reader, $coderef); it is not a method, so there is no leading - # class argument. decode_long ignores its (shifted) first argument and reads - # from the last, so pass __PACKAGE__ for it and $reader last. - my ($reader, $block_content) = @_; + # skip_block($reader, $min_bytes, $coderef); it is not a method, so there is + # no leading class argument. decode_long ignores its (shifted) first argument + # and reads from the last, so pass __PACKAGE__ for it and $reader last. + my ($reader, $min_bytes, $block_content) = @_; + # Zero-byte elements loop with no per-item input, so bound them tightly; + # other elements consume bytes (bounded by EOF) so only the structural cap + # applies. + my $limit = $min_bytes > 0 ? $MAX_COLLECTION_STRUCTURAL : $MAX_COLLECTION_ITEMS; my $block_count = decode_long(__PACKAGE__, undef, undef, $reader); + my $skipped = 0; while ($block_count) { if ($block_count < 0) { # A negative count is followed by a long block size in bytes, which @@ -364,6 +410,15 @@ sub skip_block { } } else { + # Bound the cumulative element count: skipping a huge block of + # zero-byte elements would otherwise loop unboundedly (a CPU + # exhaustion) even though it allocates nothing. + if ($skipped + $block_count > $limit) { + throw Avro::BinaryDecoder::Error::CollectionSize( + "Cannot skip a collection of more than $limit " + . "elements (declared @{[ $skipped + $block_count ]})"); + } + $skipped += $block_count; for (1..$block_count) { $block_content->(); } @@ -375,7 +430,8 @@ sub skip_block { sub skip_array { my $class = shift; my ($schema, $reader) = @_; - skip_block($reader, sub { $class->skip($schema->items, $reader) }); + skip_block($reader, _min_bytes_per_element($schema->items), + sub { $class->skip($schema->items, $reader) }); } ## 1.3.2 Arrays are encoded as a series of blocks. Each block consists of a @@ -399,7 +455,7 @@ sub decode_array { $block_size = decode_long($class, @_); ## XXX we can skip with $reader_schema? } - _ensure_collection_available($reader, $block_count, $min_bytes); + _ensure_collection_available($reader, scalar(@array), $block_count, $min_bytes); for (1..$block_count) { push @array, $class->decode( writer_schema => $writer_items, @@ -415,7 +471,8 @@ sub decode_array { sub skip_map { my $class = shift; my ($schema, $reader) = @_; - skip_block($reader, sub { + # Map entries always carry a >= 1 byte key, so pass a positive minimum. + skip_block($reader, 1 + _min_bytes_per_element($schema->values), sub { skip_string($class, $reader); $class->skip($schema->values, $reader); }); @@ -449,7 +506,7 @@ sub decode_map { $block_size = decode_long($class, @_); ## XXX we can skip with $reader_schema? } - _ensure_collection_available($reader, $block_count, $min_bytes); + _ensure_collection_available($reader, scalar(keys %hash), $block_count, $min_bytes); for (1..$block_count) { my $key = decode_string($class, @_); unless (defined $key && length $key) { @@ -543,4 +600,9 @@ sub unsigned_varint { return $int; } +package Avro::BinaryDecoder::Error::CollectionSize; +use parent -norequire, 'Error::Simple'; + +package Avro::BinaryDecoder; + 1; diff --git a/lang/perl/t/03_bin_decode.t b/lang/perl/t/03_bin_decode.t index 2c642e3edc3..0e79670963e 100644 --- a/lang/perl/t/03_bin_decode.t +++ b/lang/perl/t/03_bin_decode.t @@ -352,6 +352,162 @@ sub encode_long { is scalar(@$dec), $count, "array of nulls is not falsely rejected"; } +## Zero-byte elements (null, zero-length fixed, all-null records) consume no +## input, so the bytes-remaining check cannot bound them. A huge declared block +## count of such elements is capped instead. +sub decode_zero_byte_array { + my ($items_schema, $enc) = @_; + my $schema = Avro::Schema->parse(qq({ "type": "array", "items": $items_schema })); + open my $reader, '<', \$enc or die "Can't open memory file: $!"; + return Avro::BinaryDecoder->decode( + writer_schema => $schema, + reader_schema => $schema, + reader => $reader, + ); +} + +{ + ## The reported exploit: a ~6 byte payload declaring 200,000,000 nulls is + ## rejected by the default limit, before allocating. + throws_ok { + decode_zero_byte_array('"null"', encode_long(200_000_000) . encode_long(0)); + } qr/more than \d+ zero-byte/, "array of 200M nulls rejected by default limit"; +} + +{ + ## Within a lowered limit, a legitimate small null array still decodes. + local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000; + my $dec = decode_zero_byte_array('"null"', encode_long(1000) . encode_long(0)); + is scalar(@$dec), 1000, "array of nulls within configured limit still reads"; +} + +{ + local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000; + throws_ok { + decode_zero_byte_array('"null"', encode_long(1001) . encode_long(0)); + } qr/more than 1000 zero-byte/, "array of nulls over configured limit rejected"; +} + +{ + ## Cumulative across blocks: two blocks of 600 (1200 > 1000). + local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000; + throws_ok { + decode_zero_byte_array('"null"', encode_long(600) . encode_long(600) . encode_long(0)); + } qr/more than 1000 zero-byte/, "cumulative null blocks over limit rejected"; +} + +{ + ## A negative block count (abs value + block byte-size) is normalized and + ## must still be bounded. + local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000; + throws_ok { + decode_zero_byte_array('"null"', encode_long(-200_000) . encode_long(0)); + } qr/more than 1000 zero-byte/, "negative null block count rejected"; +} + +{ + ## A record whose only field is null encodes to zero bytes. + local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000; + throws_ok { + decode_zero_byte_array('{ "type": "record", "name": "R", "fields": [{"name":"n","type":"null"}] }', + encode_long(2000) . encode_long(0)); + } qr/more than 1000 zero-byte/, "array of all-null records over limit rejected"; +} + +{ + ## A huge map is bounded by the bytes-remaining check (each entry has a + ## >= 1 byte key), not the zero-byte cap. + my $map_schema = Avro::Schema->parse(q({ "type": "map", "values": "null" })); + my $enc = encode_long(200_000_000); + open my $reader, '<', \$enc or die "Can't open memory file: $!"; + throws_ok { + Avro::BinaryDecoder->decode( + writer_schema => $map_schema, + reader_schema => $map_schema, + reader => $reader, + ); + } qr/Collection claims/, "oversized map rejected by available-bytes check"; +} + +{ + ## Skipping a huge array writer field absent from the reader schema is + ## bounded (a CPU exhaustion otherwise), via skip_block. + local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000; + my $w_schema = Avro::Schema->parse(<<'EOJ'); + { "type": "record", "name": "test", + "fields" : [ + {"name": "arr", "type": {"type": "array", "items": "null"}}, + {"name": "a", "type": "long"} ]} +EOJ + my $r_schema = Avro::Schema->parse(<<'EOJ'); + { "type": "record", "name": "test", + "fields" : [ {"name": "a", "type": "long"} ]} +EOJ + # Hand-crafted: array block of 2000 nulls + end marker, then a = 42. + my $enc = encode_long(2000) . encode_long(0) . encode_long(42); + open my $reader, '<', \$enc or die "Can't open memory file: $!"; + throws_ok { + Avro::BinaryDecoder->decode( + writer_schema => $w_schema, + reader_schema => $r_schema, + reader => $reader, + ); + } qr/more than 1000/, "skipping oversized array field is bounded"; +} + +{ + ## The zero-byte cap raises the dedicated exception class. + local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000; + my $err; + eval { + decode_zero_byte_array('"null"', encode_long(2000) . encode_long(0)); + 1; + } or $err = $@; + isa_ok $err, 'Avro::BinaryDecoder::Error::CollectionSize', + "zero-byte cap throws CollectionSize"; +} + +{ + ## Non-zero-byte collections are bounded by a structural cap in addition to + ## the bytes-remaining check. A small backed array that passes the + ## bytes check is still rejected once its count exceeds the structural limit. + local $Avro::BinaryDecoder::MAX_COLLECTION_STRUCTURAL = 5; + my $schema = Avro::Schema->parse(q({ "type": "array", "items": "long" })); + my $enc = ''; + Avro::BinaryEncoder->encode( + schema => $schema, + data => [ 0 .. 9 ], # 10 real longs, enough bytes to pass the byte check + emit_cb => sub { $enc .= ${ $_[0] } }, + ); + open my $reader, '<', \$enc or die "Can't open memory file: $!"; + throws_ok { + Avro::BinaryDecoder->decode( + writer_schema => $schema, + reader_schema => $schema, + reader => $reader, + ); + } qr/more than 5 elements/, "non-zero collection bounded by structural cap"; +} + +{ + ## A backed array within the structural cap still decodes. + local $Avro::BinaryDecoder::MAX_COLLECTION_STRUCTURAL = 100; + my $schema = Avro::Schema->parse(q({ "type": "array", "items": "long" })); + my $enc = ''; + Avro::BinaryEncoder->encode( + schema => $schema, + data => [ 0 .. 9 ], + emit_cb => sub { $enc .= ${ $_[0] } }, + ); + open my $reader, '<', \$enc or die "Can't open memory file: $!"; + my $dec = Avro::BinaryDecoder->decode( + writer_schema => $schema, + reader_schema => $schema, + reader => $reader, + ); + is_deeply $dec, [ 0 .. 9 ], "backed array within structural cap still reads"; +} + ## Skipping writer fields absent from the reader schema must advance the reader ## by the correct relative amount (SEEK_CUR), including fixed and bytes fields. { From 384ef7ebc06be4954a6389eb20fec9757416e155 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sun, 12 Jul 2026 18:27:02 +0200 Subject: [PATCH 11/24] AVRO-4299: [perl] Test INT64_MIN block count is bounded Perl numbers do not overflow to a wrapped negative, so negating an INT64_MIN block count yields 2**63, which the existing zero-byte/structural cap already rejects. Add a regression test decoding the 10-byte INT64_MIN varint as an array block count and asserting the CollectionSize rejection, matching the negation-overflow coverage added to the C, C++, C# and PHP SDKs. Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/perl/t/03_bin_decode.t | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lang/perl/t/03_bin_decode.t b/lang/perl/t/03_bin_decode.t index 0e79670963e..944c15f92f0 100644 --- a/lang/perl/t/03_bin_decode.t +++ b/lang/perl/t/03_bin_decode.t @@ -381,6 +381,16 @@ sub decode_zero_byte_array { is scalar(@$dec), 1000, "array of nulls within configured limit still reads"; } +{ + ## INT64_MIN as a block count is the pathological negation case. Negating it + ## yields 2**63, which the cap rejects. INT64_MIN zig-zag encodes as the + ## 10-byte varint below, followed by a block byte-size (0). + my $int64_min = "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01"; + throws_ok { + decode_zero_byte_array('"null"', $int64_min . encode_long(0)); + } qr/Cannot read a collection|zero-byte|Invalid/, "INT64_MIN array block count rejected"; +} + { local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000; throws_ok { From f46ed3d75df01c915780c1651fae6c4a0b7e722f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sun, 12 Jul 2026 19:14:01 +0200 Subject: [PATCH 12/24] AVRO-4299: [perl] Apply structural cap to zero-byte collections; fix constant Addresses review feedback: - _ensure_collection_available now checks the structural cap for every collection (including zero-byte elements), before the per-element bytes check or the tighter zero-byte item cap. Previously the structural cap was only enforced for non-zero-byte elements, so raising MAX_COLLECTION_ITEMS above MAX_COLLECTION_STRUCTURAL could bypass it for zero-byte collections. - skip_block now bounds zero-byte skips by the tighter of the item and structural limits (the structural cap applies to every collection), instead of only the item cap. - DEFAULT_MAX_COLLECTION_STRUCTURAL is now (2 ** 31) - 1 - 8 = 2147483639 (Integer.MAX_VALUE - 8) instead of (2 ** 31) - 8 = 2147483640. Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/perl/lib/Avro/BinaryDecoder.pm | 31 +++++++++++++++++------------ 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/lang/perl/lib/Avro/BinaryDecoder.pm b/lang/perl/lib/Avro/BinaryDecoder.pm index d6e9ff15741..dc8702e3151 100644 --- a/lang/perl/lib/Avro/BinaryDecoder.pm +++ b/lang/perl/lib/Avro/BinaryDecoder.pm @@ -178,7 +178,7 @@ our $DEFAULT_MAX_COLLECTION_ITEMS = 10_000_000; ## Structural cap on the number of elements in any array or map (an ## overflow/defense-in-depth guard), matching the historical Integer.MAX_VALUE-8 ## limit. Non-zero-byte elements are also bounded by the bytes remaining. -our $DEFAULT_MAX_COLLECTION_STRUCTURAL = (2 ** 31) - 8; +our $DEFAULT_MAX_COLLECTION_STRUCTURAL = (2 ** 31) - 1 - 8; ## AVRO_MAX_COLLECTION_ITEMS, when set, caps both limits; otherwise zero-byte ## elements use the tighter default and all collections use the structural cap. @@ -268,6 +268,14 @@ sub _min_bytes_per_element { sub _ensure_collection_available { my ($reader, $existing, $count, $min_bytes) = @_; return if $count <= 0; + my $total = $existing + $count; + # The structural cap bounds every array/map, including zero-byte element + # collections and non-seekable readers where the bytes check cannot run. + if ($total > $MAX_COLLECTION_STRUCTURAL) { + throw Avro::BinaryDecoder::Error::CollectionSize( + "Cannot read a collection of more than $MAX_COLLECTION_STRUCTURAL " + . "elements (declared $total)"); + } if ($min_bytes > 0) { my $remaining = _bytes_remaining($reader); # Compare via integer division rather than multiplying, so $count * $min_bytes @@ -277,20 +285,13 @@ sub _ensure_collection_available { "Collection claims $count elements with at least $min_bytes bytes each, " . "but only $remaining bytes are available"); } - # Structural / overflow guard, also covering non-seekable readers where - # the bytes check above cannot run. - if ($existing + $count > $MAX_COLLECTION_STRUCTURAL) { - throw Avro::BinaryDecoder::Error::CollectionSize( - "Cannot read a collection of more than $MAX_COLLECTION_STRUCTURAL " - . "elements (declared @{[ $existing + $count ]})"); - } } # Zero-byte elements (e.g. null) consume no input, so they cannot be bounded - # by the bytes remaining; cap their cumulative count instead. - elsif ($existing + $count > $MAX_COLLECTION_ITEMS) { + # by the bytes remaining; additionally cap their cumulative count. + elsif ($total > $MAX_COLLECTION_ITEMS) { throw Avro::BinaryDecoder::Error::CollectionSize( "Cannot read a collection of more than $MAX_COLLECTION_ITEMS zero-byte " - . "elements (declared @{[ $existing + $count ]}); raise " + . "elements (declared $total); raise " . "\$Avro::BinaryDecoder::MAX_COLLECTION_ITEMS if this is legitimate"); } return; @@ -382,8 +383,12 @@ sub skip_block { my ($reader, $min_bytes, $block_content) = @_; # Zero-byte elements loop with no per-item input, so bound them tightly; # other elements consume bytes (bounded by EOF) so only the structural cap - # applies. - my $limit = $min_bytes > 0 ? $MAX_COLLECTION_STRUCTURAL : $MAX_COLLECTION_ITEMS; + # applies. Every collection is capped by the structural limit, so for + # zero-byte elements use whichever of the two limits is tighter. + my $limit = $MAX_COLLECTION_STRUCTURAL; + if ($min_bytes <= 0 && $MAX_COLLECTION_ITEMS < $limit) { + $limit = $MAX_COLLECTION_ITEMS; + } my $block_count = decode_long(__PACKAGE__, undef, undef, $reader); my $skipped = 0; while ($block_count) { From d53c40ef69cb2d25f59c440621105b11d31c89ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sun, 12 Jul 2026 20:33:55 +0200 Subject: [PATCH 13/24] AVRO-4299: [perl] Validate available bytes before skipping a fixed Addresses review feedback: skip_fixed seeked over the fixed payload without checking it is actually present. A seek past EOF silently succeeds on many handles, so a truncated input could be skipped and later decoded as zeros instead of failing. Call _ensure_available (as skip_bytes already does) before the seek, so a fixed that exceeds the bytes remaining is rejected on a size-reporting reader. Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/perl/lib/Avro/BinaryDecoder.pm | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lang/perl/lib/Avro/BinaryDecoder.pm b/lang/perl/lib/Avro/BinaryDecoder.pm index dc8702e3151..be9d38dfc1d 100644 --- a/lang/perl/lib/Avro/BinaryDecoder.pm +++ b/lang/perl/lib/Avro/BinaryDecoder.pm @@ -559,6 +559,11 @@ sub decode_union { sub skip_fixed { my $class = shift; my ($schema, $reader) = @_; + # Reject a fixed size that exceeds the bytes actually remaining before + # skipping, mirroring skip_bytes: a seek past EOF silently succeeds on many + # handles, so without this a truncated input would later read zeros instead + # of failing. + _ensure_available($reader, $schema->size); # Skip the fixed-size payload relative to the current position (SEEK_CUR); # whence 0 (SEEK_SET) would incorrectly seek to the absolute offset. A # failed seek is treated as fatal. From 929db90368f4220e3e319e5a511ddfbb976f4b89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sun, 12 Jul 2026 21:12:39 +0200 Subject: [PATCH 14/24] AVRO-4299: [perl] Force the availability check on seek-based skip paths Addresses review feedback: _ensure_available() skipped the bytes-remaining check for sizes <= 1 MiB to avoid per-value overhead. That is fine for the decode paths (they verify the actual read length), but the skip paths use seek(), which can silently succeed past EOF and verifies nothing, so a small truncated payload could be skipped and later decoded as zeros. Add a $force flag to _ensure_available and pass it from skip_bytes, skip_fixed, and skip_block's negative-block path so they always validate against the bytes remaining when the reader can report them; decode_bytes keeps the unforced (small-read) fast path. Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/perl/lib/Avro/BinaryDecoder.pm | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/lang/perl/lib/Avro/BinaryDecoder.pm b/lang/perl/lib/Avro/BinaryDecoder.pm index be9d38dfc1d..2f56604b899 100644 --- a/lang/perl/lib/Avro/BinaryDecoder.pm +++ b/lang/perl/lib/Avro/BinaryDecoder.pm @@ -132,7 +132,7 @@ sub skip_bytes { # skipping, mirroring decode_bytes: a seek past EOF silently succeeds on # many handles, so without this a truncated input would later read zeros # instead of failing. - _ensure_available($reader, $size); + _ensure_available($reader, $size, 1); # Skip forward by $size bytes relative to the current position (SEEK_CUR); # whence 0 (SEEK_SET) would incorrectly seek to the absolute offset $size. # A failed seek means the reader cannot skip and is treated as fatal. @@ -221,8 +221,12 @@ sub _bytes_remaining { } sub _ensure_available { - my ($reader, $size) = @_; - return if $size <= _MAX_UNCHECKED_READ; + my ($reader, $size, $force) = @_; + # Small reads normally skip the check to avoid per-value overhead, since the + # decode paths verify the actual read length. The skip paths use seek(), + # which can silently succeed past EOF and does not verify anything, so they + # pass $force to always validate against the bytes remaining. + return if !$force && $size <= _MAX_UNCHECKED_READ; my $remaining = _bytes_remaining($reader); return unless defined $remaining; if ($size > $remaining) { @@ -408,7 +412,7 @@ sub skip_block { # Reject a block size that exceeds the bytes actually remaining # before skipping, so a truncated input fails instead of seeking # past EOF. - _ensure_available($reader, $block_size); + _ensure_available($reader, $block_size, 1); unless ($reader->seek($block_size, 1)) { throw Avro::Schema::Error::Parse( "Failed to skip block of $block_size bytes"); @@ -563,7 +567,7 @@ sub skip_fixed { # skipping, mirroring skip_bytes: a seek past EOF silently succeeds on many # handles, so without this a truncated input would later read zeros instead # of failing. - _ensure_available($reader, $schema->size); + _ensure_available($reader, $schema->size, 1); # Skip the fixed-size payload relative to the current position (SEEK_CUR); # whence 0 (SEEK_SET) would incorrectly seek to the absolute offset. A # failed seek is treated as fatal. From f2b45621213939edf40a741b28f21f8c5ee8a434 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sun, 12 Jul 2026 21:33:48 +0200 Subject: [PATCH 15/24] AVRO-4299: [perl] Count decoded map entries for cumulative caps Addresses review feedback: decode_map used scalar(keys %hash) as the existing element count, which undercounts when duplicate keys overwrite earlier entries. A multi-block map with repeated keys could therefore exceed the structural and zero-byte cumulative caps without being rejected. Track the number of decoded entries in a separate counter and use it for the cumulative limit check. Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/perl/lib/Avro/BinaryDecoder.pm | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lang/perl/lib/Avro/BinaryDecoder.pm b/lang/perl/lib/Avro/BinaryDecoder.pm index 2f56604b899..a34df9bad00 100644 --- a/lang/perl/lib/Avro/BinaryDecoder.pm +++ b/lang/perl/lib/Avro/BinaryDecoder.pm @@ -508,6 +508,10 @@ sub decode_map { # its minimum on-wire size is 2 bytes: a 1-byte length prefix plus at least # 1 byte of key data, in addition to the value. my $min_bytes = 2 + _min_bytes_per_element($writer_values); + # Track the number of decoded entries separately: scalar(keys %hash) + # undercounts when duplicate keys overwrite earlier entries, which would let + # a multi-block map exceed the cumulative caps without being rejected. + my $decoded = 0; while ($block_count) { my $block_size; if ($block_count < 0) { @@ -515,7 +519,8 @@ sub decode_map { $block_size = decode_long($class, @_); ## XXX we can skip with $reader_schema? } - _ensure_collection_available($reader, scalar(keys %hash), $block_count, $min_bytes); + _ensure_collection_available($reader, $decoded, $block_count, $min_bytes); + $decoded += $block_count; for (1..$block_count) { my $key = decode_string($class, @_); unless (defined $key && length $key) { From 985cfa8de584ed1cfa5ff3f61ffb2ec5590c2084 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sun, 12 Jul 2026 23:17:01 +0200 Subject: [PATCH 16/24] AVRO-4299: [perl] Bounds-check union and enum indices decode_union indexed writer_schema->schemas with the decoded index without any validation, and decode_enum / skip_union only relied on an "or throw" that a negative index bypasses (Perl negative indexing wraps to a valid element), silently selecting the wrong branch/symbol. Reject an index that is negative or >= the number of branches/symbols with an Avro::Schema::Error::Parse before indexing, in decode_union, decode_enum and skip_union. Adds tests for negative and too-large union and enum indices. Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/perl/lib/Avro/BinaryDecoder.pm | 25 +++++++++++++--- lang/perl/t/03_bin_decode.t | 46 +++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/lang/perl/lib/Avro/BinaryDecoder.pm b/lang/perl/lib/Avro/BinaryDecoder.pm index a34df9bad00..59f637ba62d 100644 --- a/lang/perl/lib/Avro/BinaryDecoder.pm +++ b/lang/perl/lib/Avro/BinaryDecoder.pm @@ -371,7 +371,14 @@ sub decode_enum { my ($writer_schema, $reader_schema, $reader) = @_; my $index = decode_int($class, @_); - my $w_data = $writer_schema->symbols->[$index]; + my $symbols = $writer_schema->symbols; + ## A negative or out-of-range index is malformed; reject it before indexing + ## (Perl's negative indexing would otherwise select the wrong symbol). + if ($index < 0 || $index >= scalar @$symbols) { + throw Avro::Schema::Error::Parse( + "Enum symbol index $index out of range for " . scalar(@$symbols) . " symbols"); + } + my $w_data = $symbols->[$index]; ## 1.3.2 if the writer's symbol is not present in the reader's enum, ## then an error is signalled. throw Avro::Schema::Error::Mismatch("enum unknown") @@ -541,8 +548,11 @@ sub skip_union { my $class = shift; my ($schema, $reader) = @_; my $idx = decode_long($class, undef, undef, $reader); - my $union_schema = $schema->schemas->[$idx] - or throw Avro::Schema::Error::Parse("union union member"); + my $schemas = $schema->schemas; + if ($idx < 0 || $idx >= scalar @$schemas) { + throw Avro::Schema::Error::Parse("union union member"); + } + my $union_schema = $schemas->[$idx]; $class->skip($union_schema, $reader); } @@ -553,7 +563,14 @@ sub decode_union { my $class = shift; my ($writer_schema, $reader_schema, $reader) = @_; my $idx = decode_long($class, @_); - my $union_schema = $writer_schema->schemas->[$idx]; + my $schemas = $writer_schema->schemas; + ## A negative or out-of-range branch index is malformed; reject it before + ## indexing (skip_union performs the same check). + if ($idx < 0 || $idx >= scalar @$schemas) { + throw Avro::Schema::Error::Parse( + "Union branch index $idx out of range for " . scalar(@$schemas) . " branches"); + } + my $union_schema = $schemas->[$idx]; ## XXX TODO: schema resolution # The first schema in the reader's union that matches the selected writer's # union schema is recursively resolved against it. if none match, an error diff --git a/lang/perl/t/03_bin_decode.t b/lang/perl/t/03_bin_decode.t index 944c15f92f0..20f18c7f5f8 100644 --- a/lang/perl/t/03_bin_decode.t +++ b/lang/perl/t/03_bin_decode.t @@ -84,6 +84,52 @@ EOJ is $dec, "a", "Binary_Encodings.Complex_Types.Unions-a"; } +## union and enum index bounds +{ + my $union = Avro::Schema->parse(q(["string","null"])); + # Index 2 (zig-zag long 0x04) is out of range for a 2-branch union. + open my $reader, '<', \"\x04" or die "Can't open memory file: $!"; + throws_ok { + Avro::BinaryDecoder->decode( + writer_schema => $union, + reader_schema => $union, + reader => $reader, + ); + } 'Avro::Schema::Error::Parse', "union branch index out of range is rejected"; + + # Index -1 (zig-zag long 0x01) must not wrap to a valid branch. + open $reader, '<', \"\x01" or die "Can't open memory file: $!"; + throws_ok { + Avro::BinaryDecoder->decode( + writer_schema => $union, + reader_schema => $union, + reader => $reader, + ); + } 'Avro::Schema::Error::Parse', "negative union branch index is rejected"; + + my $enum = Avro::Schema->parse( + q({ "type": "enum", "name": "e", "symbols": [ "a", "b" ] })); + # Index 9 (zig-zag int 0x12) is out of range for a 2-symbol enum. + open $reader, '<', \"\x12" or die "Can't open memory file: $!"; + throws_ok { + Avro::BinaryDecoder->decode( + writer_schema => $enum, + reader_schema => $enum, + reader => $reader, + ); + } 'Avro::Schema::Error::Parse', "enum symbol index out of range is rejected"; + + # Index -1 (zig-zag int 0x01) must not wrap to a valid symbol. + open $reader, '<', \"\x01" or die "Can't open memory file: $!"; + throws_ok { + Avro::BinaryDecoder->decode( + writer_schema => $enum, + reader_schema => $enum, + reader => $reader, + ); + } 'Avro::Schema::Error::Parse', "negative enum symbol index is rejected"; +} + ## enum schema resolution { From afe56c521f0f5a297b2ed9344618426a3b49a9a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sun, 12 Jul 2026 23:52:37 +0200 Subject: [PATCH 17/24] AVRO-4299: [perl] Reject overlong varints in unsigned_varint unsigned_varint (used by decode_int and decode_long) looped over continuation bytes with no cap, so an overlong varint read an unbounded continuation chain and let $shift overflow into a garbage value. A 64-bit value uses at most 10 bytes; reject an 11th continuation byte with Avro::Schema::Error::Parse, matching the Java, C and C++ SDKs. Adds a regression test. Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/perl/lib/Avro/BinaryDecoder.pm | 6 ++++++ lang/perl/t/03_bin_decode.t | 16 ++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/lang/perl/lib/Avro/BinaryDecoder.pm b/lang/perl/lib/Avro/BinaryDecoder.pm index 59f637ba62d..8904e3b1cf9 100644 --- a/lang/perl/lib/Avro/BinaryDecoder.pm +++ b/lang/perl/lib/Avro/BinaryDecoder.pm @@ -626,6 +626,12 @@ sub unsigned_varint { my $more; my $shift = 0; do { + # A 64-bit value uses at most 10 bytes (shifts 0..63); reject an overlong + # varint rather than reading an unbounded continuation chain and letting + # $shift overflow into a garbage value. + if ($shift >= 70) { + throw Avro::Schema::Error::Parse("Varint is too long"); + } $reader->read(my $buf, 1); my $byte = ord $buf; my $value = $byte & 0x7F; diff --git a/lang/perl/t/03_bin_decode.t b/lang/perl/t/03_bin_decode.t index 20f18c7f5f8..e5b3f6675aa 100644 --- a/lang/perl/t/03_bin_decode.t +++ b/lang/perl/t/03_bin_decode.t @@ -130,6 +130,22 @@ EOJ } 'Avro::Schema::Error::Parse', "negative enum symbol index is rejected"; } +## overlong varint +{ + # A 64-bit value uses at most 10 bytes; an 11th continuation byte is a + # malformed overlong varint and must be rejected. + my $long = Avro::Schema->parse(q({ "type": "long" })); + my $data = ("\x80" x 10) . "\x01"; + open my $reader, '<', \$data or die "Can't open memory file: $!"; + throws_ok { + Avro::BinaryDecoder->decode( + writer_schema => $long, + reader_schema => $long, + reader => $reader, + ); + } 'Avro::Schema::Error::Parse', "overlong varint is rejected"; +} + ## enum schema resolution { From b3666f60b90c0617736376e21070e430a7e79b87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Mon, 13 Jul 2026 01:33:46 +0200 Subject: [PATCH 18/24] AVRO-4299: [perl] Treat a short read as a parse error in unsigned_varint unsigned_varint ignored the return value of $reader->read(...). On EOF/a short read $buf is empty and `ord $buf` is 0, so truncated input was silently decoded as a valid 0 byte (and could make higher-level decoders loop or allocate on corrupted values). Reject a short/undefined read with a parse error instead. Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/perl/lib/Avro/BinaryDecoder.pm | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lang/perl/lib/Avro/BinaryDecoder.pm b/lang/perl/lib/Avro/BinaryDecoder.pm index 8904e3b1cf9..d531dbfc4d8 100644 --- a/lang/perl/lib/Avro/BinaryDecoder.pm +++ b/lang/perl/lib/Avro/BinaryDecoder.pm @@ -632,7 +632,12 @@ sub unsigned_varint { if ($shift >= 70) { throw Avro::Schema::Error::Parse("Varint is too long"); } - $reader->read(my $buf, 1); + my $got = $reader->read(my $buf, 1); + # A short read (EOF) would otherwise make `ord $buf` == 0 and silently + # decode truncated input as a valid 0 byte; treat it as a parse error. + if (!$got) { + throw Avro::Schema::Error::Parse("Unexpected end of input while reading varint"); + } my $byte = ord $buf; my $value = $byte & 0x7F; $int |= $value << $shift; From 1778a637af5ad717f7d9b05e1c6baf1878c45732 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Mon, 13 Jul 2026 02:08:04 +0200 Subject: [PATCH 19/24] AVRO-4299: [perl] Bound byte-sized skip blocks; reject negative decode block size Address review feedback: - skip_block only applied the cumulative element cap to positive-count blocks. A byte-sized (negative-count) block declares an element count too, so a huge declared count with a small/zero block size (e.g. zero-byte elements) could be skipped unbounded. Apply the cumulative cap to the absolute count in that branch as well. - decode_array and decode_map read the per-block byte-size for a negative count but did not validate it; reject a negative block size (as skip_block does). Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/perl/lib/Avro/BinaryDecoder.pm | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/lang/perl/lib/Avro/BinaryDecoder.pm b/lang/perl/lib/Avro/BinaryDecoder.pm index d531dbfc4d8..261262dd5e0 100644 --- a/lang/perl/lib/Avro/BinaryDecoder.pm +++ b/lang/perl/lib/Avro/BinaryDecoder.pm @@ -404,6 +404,17 @@ sub skip_block { my $skipped = 0; while ($block_count) { if ($block_count < 0) { + # A byte-sized (negative-count) block still declares an element + # count; bound it cumulatively too, so a huge declared count with a + # small or zero block size cannot bypass the cap (e.g. zero-byte + # elements whose block size is 0). + my $count = -$block_count; + if ($skipped + $count > $limit) { + throw Avro::BinaryDecoder::Error::CollectionSize( + "Cannot skip a collection of more than $limit " + . "elements (declared @{[ $skipped + $count ]})"); + } + $skipped += $count; # A negative count is followed by a long block size in bytes, which # lets the whole block be skipped without decoding each item. Skip # forward by that many bytes relative to the current position @@ -469,6 +480,10 @@ sub decode_array { if ($block_count < 0) { $block_count = -$block_count; $block_size = decode_long($class, @_); + if ($block_size < 0) { + throw Avro::Schema::Error::Parse( + "Invalid negative array block size: $block_size"); + } ## XXX we can skip with $reader_schema? } _ensure_collection_available($reader, scalar(@array), $block_count, $min_bytes); @@ -524,6 +539,10 @@ sub decode_map { if ($block_count < 0) { $block_count = -$block_count; $block_size = decode_long($class, @_); + if ($block_size < 0) { + throw Avro::Schema::Error::Parse( + "Invalid negative map block size: $block_size"); + } ## XXX we can skip with $reader_schema? } _ensure_collection_available($reader, $decoded, $block_count, $min_bytes); From 58989e7402bb77dbb661f7950dfedde34552428f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Mon, 13 Jul 2026 02:39:05 +0200 Subject: [PATCH 20/24] AVRO-4299: [perl] Descriptive skip_union out-of-range message skip_union threw the terse "union union member" on an out-of-range branch index; use the same descriptive "Union branch index N out of range for M branches" message as decode_union. Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/perl/lib/Avro/BinaryDecoder.pm | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lang/perl/lib/Avro/BinaryDecoder.pm b/lang/perl/lib/Avro/BinaryDecoder.pm index 261262dd5e0..89105b64b06 100644 --- a/lang/perl/lib/Avro/BinaryDecoder.pm +++ b/lang/perl/lib/Avro/BinaryDecoder.pm @@ -569,7 +569,8 @@ sub skip_union { my $idx = decode_long($class, undef, undef, $reader); my $schemas = $schema->schemas; if ($idx < 0 || $idx >= scalar @$schemas) { - throw Avro::Schema::Error::Parse("union union member"); + throw Avro::Schema::Error::Parse( + "Union branch index $idx out of range for " . scalar(@$schemas) . " branches"); } my $union_schema = $schemas->[$idx]; $class->skip($union_schema, $reader); From f83a2bf769c59d27cc78388fa1eff864d2e39c06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Mon, 13 Jul 2026 03:11:05 +0200 Subject: [PATCH 21/24] AVRO-4299: [perl] Fix test comments: union index is a zig-zag int The union branch index is encoded as an Avro int per the spec; update the two test comments that said "zig-zag long". Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/perl/t/03_bin_decode.t | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lang/perl/t/03_bin_decode.t b/lang/perl/t/03_bin_decode.t index e5b3f6675aa..7b6e4299312 100644 --- a/lang/perl/t/03_bin_decode.t +++ b/lang/perl/t/03_bin_decode.t @@ -87,7 +87,7 @@ EOJ ## union and enum index bounds { my $union = Avro::Schema->parse(q(["string","null"])); - # Index 2 (zig-zag long 0x04) is out of range for a 2-branch union. + # Index 2 (zig-zag int 0x04) is out of range for a 2-branch union. open my $reader, '<', \"\x04" or die "Can't open memory file: $!"; throws_ok { Avro::BinaryDecoder->decode( @@ -97,7 +97,7 @@ EOJ ); } 'Avro::Schema::Error::Parse', "union branch index out of range is rejected"; - # Index -1 (zig-zag long 0x01) must not wrap to a valid branch. + # Index -1 (zig-zag int 0x01) must not wrap to a valid branch. open $reader, '<', \"\x01" or die "Can't open memory file: $!"; throws_ok { Avro::BinaryDecoder->decode( From fc5509ad92097052bc571dae926d97472ef71651 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Mon, 13 Jul 2026 03:53:08 +0200 Subject: [PATCH 22/24] AVRO-4299: [perl] Pin default limit in the huge-null-array test The "200M nulls rejected by default limit" test relied on the default MAX_COLLECTION_ITEMS, which BinaryDecoder.pm initializes from AVRO_MAX_COLLECTION_ITEMS at load. Localize $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS to DEFAULT_MAX_COLLECTION_ITEMS in the test so it is deterministic even if that env var is set in the runner. Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/perl/t/03_bin_decode.t | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lang/perl/t/03_bin_decode.t b/lang/perl/t/03_bin_decode.t index 7b6e4299312..82ceef9d8fb 100644 --- a/lang/perl/t/03_bin_decode.t +++ b/lang/perl/t/03_bin_decode.t @@ -430,7 +430,11 @@ sub decode_zero_byte_array { { ## The reported exploit: a ~6 byte payload declaring 200,000,000 nulls is - ## rejected by the default limit, before allocating. + ## rejected by the default limit, before allocating. Pin the limit to the + ## default so the test is deterministic even if AVRO_MAX_COLLECTION_ITEMS was + ## set in the environment when the module was loaded. + local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = + $Avro::BinaryDecoder::DEFAULT_MAX_COLLECTION_ITEMS; throws_ok { decode_zero_byte_array('"null"', encode_long(200_000_000) . encode_long(0)); } qr/more than \d+ zero-byte/, "array of 200M nulls rejected by default limit"; From 8d8a8e42c58f4dcce59f41222bc6214492c2cee0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Mon, 13 Jul 2026 06:25:50 +0200 Subject: [PATCH 23/24] AVRO-4299: [perl] skip_block size-vs-count check; make tests env-independent - skip_block's negative-count branch now also rejects a block_size too small to hold the declared element count at its minimum on-wire size, so a malformed size can't leave the reader mid-element. - BinaryDecoder initializes both collection caps from AVRO_MAX_COLLECTION_ITEMS at load, which affected the whole test suite (not just the limit tests). Reset both caps to their defaults at the top of t/03_bin_decode.t so every test is deterministic regardless of the environment (verified with the env var set to 5); individual tests still local-override where needed. - Fix the overlong-varint test comment wording. Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/perl/lib/Avro/BinaryDecoder.pm | 8 ++++++++ lang/perl/t/03_bin_decode.t | 11 ++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/lang/perl/lib/Avro/BinaryDecoder.pm b/lang/perl/lib/Avro/BinaryDecoder.pm index 89105b64b06..0fa7f8e17db 100644 --- a/lang/perl/lib/Avro/BinaryDecoder.pm +++ b/lang/perl/lib/Avro/BinaryDecoder.pm @@ -431,6 +431,14 @@ sub skip_block { # before skipping, so a truncated input fails instead of seeking # past EOF. _ensure_available($reader, $block_size, 1); + # Reject a block size too small to contain the declared element + # count (at the element's minimum on-wire size), so a malformed + # size cannot leave the reader mid-element and misalign decoding. + if ($min_bytes > 0 && $count > int($block_size / $min_bytes)) { + throw Avro::Schema::Error::Parse( + "Block size $block_size is too small for $count elements " + . "of >= $min_bytes bytes"); + } unless ($reader->seek($block_size, 1)) { throw Avro::Schema::Error::Parse( "Failed to skip block of $block_size bytes"); diff --git a/lang/perl/t/03_bin_decode.t b/lang/perl/t/03_bin_decode.t index 82ceef9d8fb..2b02c7e61a3 100644 --- a/lang/perl/t/03_bin_decode.t +++ b/lang/perl/t/03_bin_decode.t @@ -26,6 +26,15 @@ use Test::Exception; use_ok 'Avro::BinaryDecoder'; +## BinaryDecoder initializes MAX_COLLECTION_ITEMS / MAX_COLLECTION_STRUCTURAL +## from AVRO_MAX_COLLECTION_ITEMS at load time. Reset both to their defaults so +## the whole suite is deterministic regardless of the environment; individual +## tests still `local`-override them where they exercise a specific limit. +$Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = + $Avro::BinaryDecoder::DEFAULT_MAX_COLLECTION_ITEMS; +$Avro::BinaryDecoder::MAX_COLLECTION_STRUCTURAL = + $Avro::BinaryDecoder::DEFAULT_MAX_COLLECTION_STRUCTURAL; + ## spec examples { my $enc = "\x06\x66\x6f\x6f"; @@ -132,7 +141,7 @@ EOJ ## overlong varint { - # A 64-bit value uses at most 10 bytes; an 11th continuation byte is a + # A 64-bit value uses at most 10 bytes; an 11th byte (10 continuation bytes 0x80 then a terminator) is a # malformed overlong varint and must be rejected. my $long = Avro::Schema->parse(q({ "type": "long" })); my $data = ("\x80" x 10) . "\x01"; From 8c79c318f349ce78cb492b18942dc772fbb38133 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Mon, 13 Jul 2026 07:04:05 +0200 Subject: [PATCH 24/24] AVRO-4299: [perl] Correct zero-byte element test comment Fixed is not a zero-byte element type in Perl: this SDK's schema parser requires a positive Fixed size. Drop 'zero-length fixed' from the comment. Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/perl/t/03_bin_decode.t | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lang/perl/t/03_bin_decode.t b/lang/perl/t/03_bin_decode.t index 2b02c7e61a3..1aadfad79f1 100644 --- a/lang/perl/t/03_bin_decode.t +++ b/lang/perl/t/03_bin_decode.t @@ -423,9 +423,10 @@ sub encode_long { is scalar(@$dec), $count, "array of nulls is not falsely rejected"; } -## Zero-byte elements (null, zero-length fixed, all-null records) consume no -## input, so the bytes-remaining check cannot bound them. A huge declared block -## count of such elements is capped instead. +## Zero-byte elements (null, all-null records) consume no input, so the +## bytes-remaining check cannot bound them. A huge declared block count of such +## elements is capped instead. (Fixed is not zero-byte here: this SDK's schema +## parser requires a positive Fixed size.) sub decode_zero_byte_array { my ($items_schema, $enc) = @_; my $schema = Avro::Schema->parse(qq({ "type": "array", "items": $items_schema }));