From 729af4a6574d962e00d80fb0677177e4f5af5aed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sat, 11 Jul 2026 14:40:05 +0200 Subject: [PATCH 01/13] AVRO-4284: [c] Enforce a maximum decompressed block size When reading a data file, each block is decompressed according to the file's codec. A block with a very high compression ratio (or a malformed block) could expand to far more memory than its compressed size. Enforce a configurable maximum decompressed size across the deflate, snappy and lzma codecs, mirroring the Java SDK's decompression limit (AVRO-4247): snappy rejects an over-large declared length up front, while deflate and lzma cap buffer growth and reject once the output would exceed the limit. The limit defaults to 200 MiB and can be overridden with the AVRO_MAX_DECOMPRESS_LENGTH environment variable. Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/c/src/codec.c | 87 ++++++++++++++++++++++--- lang/c/tests/CMakeLists.txt | 1 + lang/c/tests/test_avro_4284.c | 118 ++++++++++++++++++++++++++++++++++ 3 files changed, 198 insertions(+), 8 deletions(-) create mode 100644 lang/c/tests/test_avro_4284.c diff --git a/lang/c/src/codec.c b/lang/c/src/codec.c index 176fb21d657..e2c3fc03b7a 100644 --- a/lang/c/src/codec.c +++ b/lang/c/src/codec.c @@ -16,6 +16,7 @@ */ #include +#include #ifdef SNAPPY_CODEC #include # if defined(__APPLE__) @@ -46,6 +47,31 @@ #define DEFAULT_BLOCK_SIZE (16 * 1024) +/* + * When reading a data file, each block is decompressed according to the file's + * codec. A block with a very high compression ratio (or a malformed block) can + * expand to far more memory than its compressed size. To guard against + * unbounded allocation, the decompressed size of a single block is capped. This + * mirrors the Java SDK's decompression limit (AVRO-4247). The default can be + * overridden with the AVRO_MAX_DECOMPRESS_LENGTH environment variable. + */ +#define AVRO_DEFAULT_MAX_DECOMPRESS_LENGTH ((int64_t) 200 * 1024 * 1024) /* 200 MiB */ + +static int64_t +avro_max_decompress_length(void) +{ + const char *env = getenv("AVRO_MAX_DECOMPRESS_LENGTH"); + if (env != NULL && *env != '\0') { + char *end = NULL; + long long value = strtoll(env, &end, 10); + if (end != NULL && *end == '\0' && value > 0) { + return (int64_t) value; + } + } + return AVRO_DEFAULT_MAX_DECOMPRESS_LENGTH; +} + + /* NULL codec */ static int @@ -147,6 +173,12 @@ static int decode_snappy(avro_codec_t c, void * data, int64_t len) return 1; } + if ((int64_t) outlen > avro_max_decompress_length()) { + avro_set_error("Decompressed block size %llu exceeds the maximum allowed of %lld bytes", + (unsigned long long) outlen, (long long) avro_max_decompress_length()); + return 1; + } + if (!c->block_data) { c->block_data = avro_malloc(outlen); c->block_size = outlen; @@ -304,6 +336,7 @@ static int encode_deflate(avro_codec_t c, void * data, int64_t len) static int decode_deflate(avro_codec_t c, void * data, int64_t len) { int err; + int64_t max_len = avro_max_decompress_length(); z_stream *s = codec_data_inflate_stream(c->codec_data); if (!c->block_data) { @@ -331,6 +364,15 @@ static int decode_deflate(avro_codec_t c, void * data, int64_t len) { err = inflate(s, Z_FINISH); + // Reject a block that decompresses to more than the allowed maximum, + // to guard against unbounded allocation from a high-ratio block. + if ((int64_t) s->total_out > max_len) { + inflateEnd(s); + avro_set_error("Decompressed block size exceeds the maximum allowed of %lld bytes", + (long long) max_len); + return 1; + } + // Apparently if there is yet available space in the output then something // has gone wrong in decompressing the data (according to cpython zlibmodule.c) if (err == Z_BUF_ERROR && s->avail_out > 0) { @@ -339,13 +381,24 @@ static int decode_deflate(avro_codec_t c, void * data, int64_t len) return 1; } - // The buffer was not big enough. resize it. + // The buffer was not big enough. resize it, without growing beyond + // the configured maximum decompressed size. if (err == Z_BUF_ERROR) { - c->block_data = avro_realloc(c->block_data, c->block_size, c->block_size * 2); + int64_t new_size = c->block_size * 2; + if (new_size > max_len) { + new_size = max_len; + } + if (new_size <= c->block_size) { + inflateEnd(s); + avro_set_error("Decompressed block size exceeds the maximum allowed of %lld bytes", + (long long) max_len); + return 1; + } + c->block_data = avro_realloc(c->block_data, c->block_size, new_size); s->next_out = c->block_data + s->total_out; - s->avail_out += c->block_size; - c->block_size = c->block_size * 2; + s->avail_out += (uInt)(new_size - c->block_size); + c->block_size = new_size; } } while (err == Z_BUF_ERROR); @@ -462,6 +515,7 @@ static int decode_lzma(avro_codec_t codec, void * data, int64_t len) { size_t read_pos = 0; size_t write_pos = 0; + int64_t max_len = avro_max_decompress_length(); lzma_ret ret; lzma_filter* filters = codec_data_lzma_filters(codec->codec_data); @@ -483,11 +537,28 @@ static int decode_lzma(avro_codec_t codec, void * data, int64_t len) codec->used_size = write_pos; - // If it ran out of space to decode, give it more!! - // It will continue where it left off because of read_pos and write_pos. + // Reject a block that decompresses to more than the allowed maximum, + // to guard against unbounded allocation from a high-ratio block. + if ((int64_t) write_pos > max_len) { + avro_set_error("Decompressed block size exceeds the maximum allowed of %lld bytes", + (long long) max_len); + return 1; + } + + // If it ran out of space to decode, give it more (without growing + // beyond the configured maximum decompressed size). if (ret == LZMA_BUF_ERROR) { - codec->block_data = avro_realloc(codec->block_data, codec->block_size, codec->block_size * 2); - codec->block_size = codec->block_size * 2; + int64_t new_size = codec->block_size * 2; + if (new_size > max_len) { + new_size = max_len; + } + if (new_size <= codec->block_size) { + avro_set_error("Decompressed block size exceeds the maximum allowed of %lld bytes", + (long long) max_len); + return 1; + } + codec->block_data = avro_realloc(codec->block_data, codec->block_size, new_size); + codec->block_size = new_size; } } while (ret == LZMA_BUF_ERROR); diff --git a/lang/c/tests/CMakeLists.txt b/lang/c/tests/CMakeLists.txt index 6b4164fa740..08ec312692c 100644 --- a/lang/c/tests/CMakeLists.txt +++ b/lang/c/tests/CMakeLists.txt @@ -88,3 +88,4 @@ add_avro_test_checkmem(test_avro_1691) add_avro_test_checkmem(test_avro_1906) add_avro_test_checkmem(test_avro_1904) add_avro_test_checkmem(test_avro_4246) +add_avro_test_checkmem(test_avro_4284) diff --git a/lang/c/tests/test_avro_4284.c b/lang/c/tests/test_avro_4284.c new file mode 100644 index 00000000000..bcbdb1f60a8 --- /dev/null +++ b/lang/c/tests/test_avro_4284.c @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +/* + * AVRO-4284: a data-file block is decompressed according to the file's codec. + * A block with a very high compression ratio (or a malformed block) can expand + * to far more memory than its compressed size. Decompression must reject a + * block whose decompressed size would exceed the configured maximum, which + * these tests set to a small value via AVRO_MAX_DECOMPRESS_LENGTH. + */ + +#include +#include +#include +#include + +#include "codec.h" + +/* Size of the highly compressible payload used to build an over-large block. */ +#define PAYLOAD_SIZE (4 * 1024 * 1024) /* 4 MiB of zeros */ +/* Decompression limit for the test, smaller than the payload. */ +#define TEST_LIMIT "1048576" /* 1 MiB */ + +/* + * Compress `payload` with the named codec and attempt to decompress it with a + * decompression limit smaller than the payload. Returns 0 if the codec is not + * available (test skipped for that codec), 1 on test failure, 2 on success. + */ +static int +check_codec_rejects_oversized(const char *name, const char *payload, int64_t payload_len) +{ + struct avro_codec_t_ codec; + memset(&codec, 0, sizeof(codec)); + + if (avro_codec(&codec, name) != 0) { + fprintf(stderr, " codec %s not available, skipping\n", name); + return 0; + } + + if (avro_codec_encode(&codec, (void *) payload, payload_len) != 0) { + fprintf(stderr, " codec %s: encode failed: %s\n", name, avro_strerror()); + avro_codec_reset(&codec); + return 1; + } + + /* Copy the compressed bytes; decode reuses the codec's block buffer. */ + int64_t compressed_len = codec.used_size; + char *compressed = (char *) malloc(compressed_len); + if (compressed == NULL) { + avro_codec_reset(&codec); + return 1; + } + memcpy(compressed, codec.block_data, compressed_len); + + int rc = avro_codec_decode(&codec, compressed, compressed_len); + + free(compressed); + avro_codec_reset(&codec); + + if (rc == 0) { + fprintf(stderr, " codec %s: expected decompression to be rejected but it succeeded\n", name); + return 1; + } + fprintf(stderr, " codec %s: over-limit block rejected as expected\n", name); + return 2; +} + +int main(void) +{ +#ifdef _WIN32 + _putenv_s("AVRO_MAX_DECOMPRESS_LENGTH", TEST_LIMIT); +#else + setenv("AVRO_MAX_DECOMPRESS_LENGTH", TEST_LIMIT, 1); +#endif + + char *payload = (char *) calloc(1, PAYLOAD_SIZE); + if (payload == NULL) { + fprintf(stderr, "Cannot allocate test payload\n"); + exit(EXIT_FAILURE); + } + + int checked = 0; + const char *codecs[] = { "deflate", "snappy", "lzma" }; + size_t i; + for (i = 0; i < sizeof(codecs) / sizeof(codecs[0]); i++) { + int r = check_codec_rejects_oversized(codecs[i], payload, PAYLOAD_SIZE); + if (r == 1) { + free(payload); + exit(EXIT_FAILURE); + } + if (r == 2) { + checked++; + } + } + + free(payload); + + if (checked == 0) { + fprintf(stderr, "No compression codecs available; nothing exercised\n"); + /* Not a failure: the build may have been configured without codecs. */ + } + + exit(EXIT_SUCCESS); +} From 3aeb3265f80e78550903deadbab860ed876cb010 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sat, 11 Jul 2026 17:23:21 +0200 Subject: [PATCH 02/13] AVRO-4284: [c] Address review: snappy length guard and NULL-safe realloc - decode_snappy rejects a block shorter than the 4-byte CRC suffix before computing len - 4 (which would underflow), and caches the decompression limit instead of re-parsing the environment in the error path. - decode_deflate and decode_lzma realloc into a temporary pointer and fail cleanly on OOM instead of null-dereferencing and leaking the old buffer. - The test casts the codec's used_size to size_t for malloc/memcpy and guards against a non-positive length. Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/c/src/codec.c | 30 +++++++++++++++++++++++++----- lang/c/tests/test_avro_4284.c | 10 ++++++++-- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/lang/c/src/codec.c b/lang/c/src/codec.c index e2c3fc03b7a..36d1f45b244 100644 --- a/lang/c/src/codec.c +++ b/lang/c/src/codec.c @@ -167,15 +167,24 @@ static int decode_snappy(avro_codec_t c, void * data, int64_t len) { uint32_t crc; size_t outlen; + int64_t max_len = avro_max_decompress_length(); + + /* The block carries a trailing 4-byte CRC; a length below that is + * malformed and would underflow len - 4. */ + if (len < 4) { + avro_set_error("Snappy block too small (%lld bytes), expected at least 4", + (long long) len); + return 1; + } if (snappy_uncompressed_length((const char*)data, len-4, &outlen) != SNAPPY_OK) { avro_set_error("Uncompressed length error in snappy"); return 1; } - if ((int64_t) outlen > avro_max_decompress_length()) { + if ((int64_t) outlen > max_len) { avro_set_error("Decompressed block size %llu exceeds the maximum allowed of %lld bytes", - (unsigned long long) outlen, (long long) avro_max_decompress_length()); + (unsigned long long) outlen, (long long) max_len); return 1; } @@ -395,8 +404,14 @@ static int decode_deflate(avro_codec_t c, void * data, int64_t len) (long long) max_len); return 1; } - c->block_data = avro_realloc(c->block_data, c->block_size, new_size); - s->next_out = c->block_data + s->total_out; + void *new_data = avro_realloc(c->block_data, c->block_size, new_size); + if (new_data == NULL) { + inflateEnd(s); + avro_set_error("Cannot allocate memory for deflate"); + return 1; + } + c->block_data = new_data; + s->next_out = (Bytef *) c->block_data + s->total_out; s->avail_out += (uInt)(new_size - c->block_size); c->block_size = new_size; } @@ -557,7 +572,12 @@ static int decode_lzma(avro_codec_t codec, void * data, int64_t len) (long long) max_len); return 1; } - codec->block_data = avro_realloc(codec->block_data, codec->block_size, new_size); + void *new_data = avro_realloc(codec->block_data, codec->block_size, new_size); + if (new_data == NULL) { + avro_set_error("Cannot allocate memory for lzma decoder"); + return 1; + } + codec->block_data = new_data; codec->block_size = new_size; } diff --git a/lang/c/tests/test_avro_4284.c b/lang/c/tests/test_avro_4284.c index bcbdb1f60a8..b3eee843ebf 100644 --- a/lang/c/tests/test_avro_4284.c +++ b/lang/c/tests/test_avro_4284.c @@ -59,12 +59,18 @@ check_codec_rejects_oversized(const char *name, const char *payload, int64_t pay /* Copy the compressed bytes; decode reuses the codec's block buffer. */ int64_t compressed_len = codec.used_size; - char *compressed = (char *) malloc(compressed_len); + if (compressed_len <= 0) { + fprintf(stderr, " codec %s: unexpected compressed length %lld\n", + name, (long long) compressed_len); + avro_codec_reset(&codec); + return 1; + } + char *compressed = (char *) malloc((size_t) compressed_len); if (compressed == NULL) { avro_codec_reset(&codec); return 1; } - memcpy(compressed, codec.block_data, compressed_len); + memcpy(compressed, codec.block_data, (size_t) compressed_len); int rc = avro_codec_decode(&codec, compressed, compressed_len); From 65d4dcf14edfb121942b64b2c797039c4cc29efa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sat, 11 Jul 2026 18:54:01 +0200 Subject: [PATCH 03/13] AVRO-4284: [c] Compare snappy length unsigned; assert size-limit error - decode_snappy compared (int64_t) outlen > max_len. A declared length above INT64_MAX would wrap to a negative int64_t and slip past the cap; compare as unsigned 64-bit instead. - The test now captures avro_strerror() and asserts the rejection is specifically the 'exceeds the maximum' size-limit error, not an unrelated failure. Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/c/src/codec.c | 4 +++- lang/c/tests/test_avro_4284.c | 13 +++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/lang/c/src/codec.c b/lang/c/src/codec.c index 36d1f45b244..4f8eed9d0dd 100644 --- a/lang/c/src/codec.c +++ b/lang/c/src/codec.c @@ -182,7 +182,9 @@ static int decode_snappy(avro_codec_t c, void * data, int64_t len) return 1; } - if ((int64_t) outlen > max_len) { + /* Compare as unsigned 64-bit: casting outlen (size_t) to int64_t could + * wrap negative for a declared length above INT64_MAX and bypass the cap. */ + if ((uint64_t) outlen > (uint64_t) max_len) { avro_set_error("Decompressed block size %llu exceeds the maximum allowed of %lld bytes", (unsigned long long) outlen, (long long) max_len); return 1; diff --git a/lang/c/tests/test_avro_4284.c b/lang/c/tests/test_avro_4284.c index b3eee843ebf..c1be1f47274 100644 --- a/lang/c/tests/test_avro_4284.c +++ b/lang/c/tests/test_avro_4284.c @@ -73,6 +73,13 @@ check_codec_rejects_oversized(const char *name, const char *payload, int64_t pay memcpy(compressed, codec.block_data, (size_t) compressed_len); int rc = avro_codec_decode(&codec, compressed, compressed_len); + /* Capture the error before reset/free may clobber it. */ + char err_copy[512]; + err_copy[0] = '\0'; + if (rc != 0) { + strncpy(err_copy, avro_strerror(), sizeof(err_copy) - 1); + err_copy[sizeof(err_copy) - 1] = '\0'; + } free(compressed); avro_codec_reset(&codec); @@ -81,6 +88,12 @@ check_codec_rejects_oversized(const char *name, const char *payload, int64_t pay fprintf(stderr, " codec %s: expected decompression to be rejected but it succeeded\n", name); return 1; } + /* Ensure it was rejected specifically for exceeding the size limit, not + * some unrelated failure (e.g. a CRC error or regression). */ + if (strstr(err_copy, "exceeds the maximum") == NULL) { + fprintf(stderr, " codec %s: rejected but not for the size limit: %s\n", name, err_copy); + return 1; + } fprintf(stderr, " codec %s: over-limit block rejected as expected\n", name); return 2; } From d280eaf0475cf7f33b086c423483bc90a047db73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sat, 11 Jul 2026 19:30:16 +0200 Subject: [PATCH 04/13] AVRO-4284: [c] Clamp decompress cap and guard buffer-growth overflow - avro_max_decompress_length() clamps the parsed value to SIZE_MAX (and now checks errno), so a configured limit larger than size_t cannot truncate when passed to avro_malloc/avro_realloc on 32-bit platforms. - decode_deflate caps its working limit to UINT_MAX so the avail_out update (a uInt) cannot truncate and leave zlib's state inconsistent. - decode_deflate and decode_lzma now double the output buffer with a checked pattern (compare against max_len/2 before multiplying) to avoid signed overflow when the buffer has already grown near INT64_MAX/2. Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/c/src/codec.c | 41 +++++++++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/lang/c/src/codec.c b/lang/c/src/codec.c index 4f8eed9d0dd..b6be061d651 100644 --- a/lang/c/src/codec.c +++ b/lang/c/src/codec.c @@ -44,6 +44,9 @@ #include "avro/errors.h" #include "avro/allocation.h" #include "codec.h" +#include +#include +#include #define DEFAULT_BLOCK_SIZE (16 * 1024) @@ -63,9 +66,18 @@ avro_max_decompress_length(void) const char *env = getenv("AVRO_MAX_DECOMPRESS_LENGTH"); if (env != NULL && *env != '\0') { char *end = NULL; + errno = 0; long long value = strtoll(env, &end, 10); - if (end != NULL && *end == '\0' && value > 0) { - return (int64_t) value; + if (errno == 0 && end != NULL && *end == '\0' && value > 0) { + int64_t v = (int64_t) value; + /* Clamp to what size_t can address so the value is always + * safe to use as an allocation cap. On 32-bit platforms + * size_t is narrower than int64_t, and an unclamped value + * would truncate when passed to avro_malloc/avro_realloc. */ + if ((uint64_t) v > (uint64_t) SIZE_MAX) { + return (int64_t) SIZE_MAX; + } + return v; } } return AVRO_DEFAULT_MAX_DECOMPRESS_LENGTH; @@ -350,6 +362,13 @@ static int decode_deflate(avro_codec_t c, void * data, int64_t len) int64_t max_len = avro_max_decompress_length(); z_stream *s = codec_data_inflate_stream(c->codec_data); + /* zlib's avail_out is a uInt; keep the working cap within its range so + * the buffer-growth arithmetic below cannot truncate when updating + * avail_out, which would leave zlib's state inconsistent. */ + if (max_len > (int64_t) UINT_MAX) { + max_len = (int64_t) UINT_MAX; + } + if (!c->block_data) { c->block_data = avro_malloc(DEFAULT_BLOCK_SIZE); c->block_size = DEFAULT_BLOCK_SIZE; @@ -396,9 +415,14 @@ static int decode_deflate(avro_codec_t c, void * data, int64_t len) // the configured maximum decompressed size. if (err == Z_BUF_ERROR) { - int64_t new_size = c->block_size * 2; - if (new_size > max_len) { + // Double the buffer, guarding against signed overflow when + // block_size is already large, and never growing beyond the + // configured maximum decompressed size. + int64_t new_size; + if (c->block_size > max_len / 2) { new_size = max_len; + } else { + new_size = c->block_size * 2; } if (new_size <= c->block_size) { inflateEnd(s); @@ -565,9 +589,14 @@ static int decode_lzma(avro_codec_t codec, void * data, int64_t len) // If it ran out of space to decode, give it more (without growing // beyond the configured maximum decompressed size). if (ret == LZMA_BUF_ERROR) { - int64_t new_size = codec->block_size * 2; - if (new_size > max_len) { + // Double the buffer, guarding against signed overflow when + // block_size is already large, and never growing beyond the + // configured maximum decompressed size. + int64_t new_size; + if (codec->block_size > max_len / 2) { new_size = max_len; + } else { + new_size = codec->block_size * 2; } if (new_size <= codec->block_size) { avro_set_error("Decompressed block size exceeds the maximum allowed of %lld bytes", From ae902c286d6d638d11e1850c968b5339ae0f79cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Mon, 13 Jul 2026 01:28:00 +0200 Subject: [PATCH 05/13] AVRO-4284: [c] Clamp initial decode buffer to the decompression cap; check setenv Address review feedback: - decode_zlib/decode_lzma allocated the initial output buffer at DEFAULT_BLOCK_SIZE regardless of the configured cap; if AVRO_MAX_DECOMPRESS_LENGTH is set below DEFAULT_BLOCK_SIZE the initial allocation exceeded the limit. Allocate min(DEFAULT_BLOCK_SIZE, max_len). - test_avro_4284: check the setenv/_putenv_s return value and fail early if the environment variable could not be set, so the test can't silently run with the default limit. Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/c/src/codec.c | 22 ++++++++++++++++++---- lang/c/tests/test_avro_4284.c | 10 ++++++++-- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/lang/c/src/codec.c b/lang/c/src/codec.c index b6be061d651..77f0463a4c6 100644 --- a/lang/c/src/codec.c +++ b/lang/c/src/codec.c @@ -370,8 +370,15 @@ static int decode_deflate(avro_codec_t c, void * data, int64_t len) } if (!c->block_data) { - c->block_data = avro_malloc(DEFAULT_BLOCK_SIZE); - c->block_size = DEFAULT_BLOCK_SIZE; + /* Do not allocate more than the configured decompression cap: if + * max_len is smaller than DEFAULT_BLOCK_SIZE, start at max_len. */ + size_t init_size = ((int64_t) DEFAULT_BLOCK_SIZE > max_len) + ? (size_t) max_len : (size_t) DEFAULT_BLOCK_SIZE; + if (init_size == 0) { + init_size = 1; + } + c->block_data = avro_malloc(init_size); + c->block_size = init_size; } if (!c->block_data) @@ -561,8 +568,15 @@ static int decode_lzma(avro_codec_t codec, void * data, int64_t len) lzma_filter* filters = codec_data_lzma_filters(codec->codec_data); if (!codec->block_data) { - codec->block_data = avro_malloc(DEFAULT_BLOCK_SIZE); - codec->block_size = DEFAULT_BLOCK_SIZE; + /* Do not allocate more than the configured decompression cap: if + * max_len is smaller than DEFAULT_BLOCK_SIZE, start at max_len. */ + size_t init_size = ((int64_t) DEFAULT_BLOCK_SIZE > max_len) + ? (size_t) max_len : (size_t) DEFAULT_BLOCK_SIZE; + if (init_size == 0) { + init_size = 1; + } + codec->block_data = avro_malloc(init_size); + codec->block_size = init_size; } if (!codec->block_data) { diff --git a/lang/c/tests/test_avro_4284.c b/lang/c/tests/test_avro_4284.c index c1be1f47274..ad3585e1b13 100644 --- a/lang/c/tests/test_avro_4284.c +++ b/lang/c/tests/test_avro_4284.c @@ -101,9 +101,15 @@ check_codec_rejects_oversized(const char *name, const char *payload, int64_t pay int main(void) { #ifdef _WIN32 - _putenv_s("AVRO_MAX_DECOMPRESS_LENGTH", TEST_LIMIT); + if (_putenv_s("AVRO_MAX_DECOMPRESS_LENGTH", TEST_LIMIT) != 0) { + fprintf(stderr, "Cannot set AVRO_MAX_DECOMPRESS_LENGTH\n"); + exit(EXIT_FAILURE); + } #else - setenv("AVRO_MAX_DECOMPRESS_LENGTH", TEST_LIMIT, 1); + if (setenv("AVRO_MAX_DECOMPRESS_LENGTH", TEST_LIMIT, 1) != 0) { + fprintf(stderr, "Cannot set AVRO_MAX_DECOMPRESS_LENGTH\n"); + exit(EXIT_FAILURE); + } #endif char *payload = (char *) calloc(1, PAYLOAD_SIZE); From 8ca55781185172f6bd003d6ee3b1572170467517 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Mon, 13 Jul 2026 02:01:39 +0200 Subject: [PATCH 06/13] AVRO-4284: [c] Clamp AVRO_MAX_DECOMPRESS_LENGTH to INT64_MAX before narrowing avro_max_decompress_length cast the strtoll (long long) result to int64_t before range-validating it. long long may be wider than 64-bit, so a large value could overflow/wrap in the cast (potentially becoming negative before the SIZE_MAX clamp). Clamp the long long value to INT64_MAX before narrowing to int64_t. Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/c/src/codec.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lang/c/src/codec.c b/lang/c/src/codec.c index 77f0463a4c6..cb04d1a444d 100644 --- a/lang/c/src/codec.c +++ b/lang/c/src/codec.c @@ -69,6 +69,12 @@ avro_max_decompress_length(void) errno = 0; long long value = strtoll(env, &end, 10); if (errno == 0 && end != NULL && *end == '\0' && value > 0) { + /* Clamp to INT64_MAX before narrowing to int64_t: long + * long may be wider than 64-bit, so a large (but valid) + * value could otherwise overflow/wrap in the cast. */ + if (value > (long long) INT64_MAX) { + value = (long long) INT64_MAX; + } int64_t v = (int64_t) value; /* Clamp to what size_t can address so the value is always * safe to use as an allocation cap. On 32-bit platforms From 6f65bbaf9f7f0a7efc64332d2552c3ec0538480f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Mon, 13 Jul 2026 02:41:25 +0200 Subject: [PATCH 07/13] AVRO-4284: [c] Compare lzma decompressed size in size_t, not narrowed int64_t decode_lzma's decompressed-size guard cast write_pos (size_t) to int64_t; if write_pos exceeded INT64_MAX the cast could wrap negative and bypass the cap. Compare in size_t against max_len (which is >= 0 and clamped to SIZE_MAX) instead. Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/c/src/codec.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lang/c/src/codec.c b/lang/c/src/codec.c index cb04d1a444d..015ad70112f 100644 --- a/lang/c/src/codec.c +++ b/lang/c/src/codec.c @@ -600,7 +600,10 @@ static int decode_lzma(avro_codec_t codec, void * data, int64_t len) // Reject a block that decompresses to more than the allowed maximum, // to guard against unbounded allocation from a high-ratio block. - if ((int64_t) write_pos > max_len) { + // Compare in size_t (max_len is >= 0 and clamped to SIZE_MAX) rather + // than narrowing write_pos to int64_t, which could wrap negative and + // bypass the cap if write_pos ever exceeds INT64_MAX. + if (write_pos > (size_t) max_len) { avro_set_error("Decompressed block size exceeds the maximum allowed of %lld bytes", (long long) max_len); return 1; From 0db06f8a17f8e5e581b367986a8bc7dde81ad99a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Mon, 13 Jul 2026 02:50:41 +0200 Subject: [PATCH 08/13] AVRO-4284: [c] Clamp AVRO_MAX_DECOMPRESS_LENGTH on strtoll overflow Previously an out-of-range value (strtoll ERANGE) fell back to the 200 MiB default, so setting the env var just above LLONG_MAX surprisingly *reduced* the effective limit. Treat positive overflow (ERANGE with LLONG_MAX) as a request for the maximum and clamp to the largest usable cap (min of INT64_MAX and SIZE_MAX). Also simplified the in-range clamp to the same max_cap. Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/c/src/codec.c | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/lang/c/src/codec.c b/lang/c/src/codec.c index 015ad70112f..0a433a376ec 100644 --- a/lang/c/src/codec.c +++ b/lang/c/src/codec.c @@ -68,22 +68,25 @@ avro_max_decompress_length(void) char *end = NULL; errno = 0; long long value = strtoll(env, &end, 10); - if (errno == 0 && end != NULL && *end == '\0' && value > 0) { - /* Clamp to INT64_MAX before narrowing to int64_t: long - * long may be wider than 64-bit, so a large (but valid) - * value could otherwise overflow/wrap in the cast. */ - if (value > (long long) INT64_MAX) { - value = (long long) INT64_MAX; + if (end != NULL && *end == '\0') { + /* The largest value usable as an allocation cap: the + * smaller of INT64_MAX and SIZE_MAX. */ + int64_t max_cap = ((uint64_t) SIZE_MAX < (uint64_t) INT64_MAX) + ? (int64_t) SIZE_MAX : INT64_MAX; + /* strtoll returns LLONG_MAX and sets ERANGE on positive + * overflow. A value above the representable range is still + * a request for a very large "max", so clamp to the + * maximum rather than falling back to the (smaller) + * default. */ + if (errno == ERANGE && value == LLONG_MAX) { + return max_cap; } - int64_t v = (int64_t) value; - /* Clamp to what size_t can address so the value is always - * safe to use as an allocation cap. On 32-bit platforms - * size_t is narrower than int64_t, and an unclamped value - * would truncate when passed to avro_malloc/avro_realloc. */ - if ((uint64_t) v > (uint64_t) SIZE_MAX) { - return (int64_t) SIZE_MAX; + if (errno == 0 && value > 0) { + if (value > (long long) max_cap) { + return max_cap; + } + return (int64_t) value; } - return v; } } return AVRO_DEFAULT_MAX_DECOMPRESS_LENGTH; From 9f97d0cea3cf0f0ebbc46105cacc8e308d5c7437 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Mon, 13 Jul 2026 03:06:26 +0200 Subject: [PATCH 09/13] AVRO-4284: [c] Range-check compressed length before zlib avail_in decode_deflate assigned the int64_t compressed block length directly into zlib's uInt avail_in, which would truncate for a length above UINT_MAX and cause incorrect/incomplete decompression. Reject a negative or > UINT_MAX length and cast explicitly. Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/c/src/codec.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lang/c/src/codec.c b/lang/c/src/codec.c index 0a433a376ec..18486de20f7 100644 --- a/lang/c/src/codec.c +++ b/lang/c/src/codec.c @@ -398,8 +398,16 @@ static int decode_deflate(avro_codec_t c, void * data, int64_t len) c->used_size = 0; + /* zlib's avail_in is a uInt; a compressed length above UINT_MAX would + * truncate and cause incorrect/incomplete decompression. Reject it. */ + if (len < 0 || len > (int64_t) UINT_MAX) { + avro_set_error("Compressed block size %lld is out of range for deflate", + (long long) len); + return 1; + } + s->next_in = data; - s->avail_in = len; + s->avail_in = (uInt) len; s->next_out = c->block_data; s->avail_out = c->block_size; From 6c915e6438c1a8834a277052694066326afa8d9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Mon, 13 Jul 2026 03:30:42 +0200 Subject: [PATCH 10/13] AVRO-4284: [c] Range-check compressed length before liblzma decode decode_lzma passed the int64_t compressed length directly to lzma_raw_buffer_decode's size_t input-size parameter; a negative value converts to a huge size_t and a value above SIZE_MAX truncates. Reject a negative or > SIZE_MAX length before decoding. Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/c/src/codec.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lang/c/src/codec.c b/lang/c/src/codec.c index 18486de20f7..c03216a0abf 100644 --- a/lang/c/src/codec.c +++ b/lang/c/src/codec.c @@ -601,6 +601,15 @@ static int decode_lzma(avro_codec_t codec, void * data, int64_t len) return 1; } + /* len is the compressed input size, passed to liblzma as a size_t. A + * negative value would convert to a huge size_t, and a value above + * SIZE_MAX would truncate; reject both. */ + if (len < 0 || (uint64_t) len > (uint64_t) SIZE_MAX) { + avro_set_error("Compressed block size %lld is out of range for lzma", + (long long) len); + return 1; + } + do { ret = lzma_raw_buffer_decode(filters, NULL, data, From 336d3d96850dc7153df8103d6837b404b5e23925 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Mon, 13 Jul 2026 03:58:14 +0200 Subject: [PATCH 11/13] AVRO-4284: [c] Reset (not end) the deflate stream on the over-limit path The over-limit rejection paths called inflateEnd(s), permanently tearing down the codec's shared zlib inflate stream, so a subsequent avro_codec_decode() with the same codec would fail. Use inflateReset(s) instead so the codec stays reusable after an over-large block is rejected (the stream itself is healthy; we're only refusing the output). Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/c/src/codec.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lang/c/src/codec.c b/lang/c/src/codec.c index c03216a0abf..2f3857957e3 100644 --- a/lang/c/src/codec.c +++ b/lang/c/src/codec.c @@ -421,7 +421,9 @@ static int decode_deflate(avro_codec_t c, void * data, int64_t len) // Reject a block that decompresses to more than the allowed maximum, // to guard against unbounded allocation from a high-ratio block. if ((int64_t) s->total_out > max_len) { - inflateEnd(s); + // Reset (not end) the stream so the codec stays reusable for + // subsequent blocks rather than being permanently torn down. + inflateReset(s); avro_set_error("Decompressed block size exceeds the maximum allowed of %lld bytes", (long long) max_len); return 1; @@ -449,7 +451,7 @@ static int decode_deflate(avro_codec_t c, void * data, int64_t len) new_size = c->block_size * 2; } if (new_size <= c->block_size) { - inflateEnd(s); + inflateReset(s); avro_set_error("Decompressed block size exceeds the maximum allowed of %lld bytes", (long long) max_len); return 1; From bf50c30e4adb64f43d06dfd4add5b76a7f635da3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Mon, 13 Jul 2026 06:43:50 +0200 Subject: [PATCH 12/13] AVRO-4284: [c] Only skip the codec test when the codec is not compiled in check_codec_rejects_oversized() skipped on any avro_codec() failure, which could hide a real regression (a compiled-in codec whose initialization fails). Skip only when the error is "Unknown codec ..." (the not-compiled-in case); any other initialization failure now fails the test with the reported error. Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/c/tests/test_avro_4284.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/lang/c/tests/test_avro_4284.c b/lang/c/tests/test_avro_4284.c index ad3585e1b13..6ba1a4bc929 100644 --- a/lang/c/tests/test_avro_4284.c +++ b/lang/c/tests/test_avro_4284.c @@ -47,8 +47,17 @@ check_codec_rejects_oversized(const char *name, const char *payload, int64_t pay memset(&codec, 0, sizeof(codec)); if (avro_codec(&codec, name) != 0) { - fprintf(stderr, " codec %s not available, skipping\n", name); - return 0; + /* A codec that is not compiled in reports "Unknown codec ..."; + * only skip in that case. Any other failure (e.g. a compiled-in + * codec whose initialization failed) is a real regression and + * must fail the test rather than be silently skipped. */ + const char *err = avro_strerror(); + if (err != NULL && strstr(err, "Unknown codec") != NULL) { + fprintf(stderr, " codec %s not available, skipping\n", name); + return 0; + } + fprintf(stderr, " codec %s: initialization failed: %s\n", name, err); + return 1; } if (avro_codec_encode(&codec, (void *) payload, payload_len) != 0) { From 33dc44573c21ab7d822d02e5dbf94e20ace95573 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Mon, 13 Jul 2026 07:06:49 +0200 Subject: [PATCH 13/13] AVRO-4284: [c] Range-check the snappy payload length before size_t conversion decode_snappy passed `len - 4` (int64_t) to snappy_uncompressed_length / snappy_uncompress (which take size_t) and used it in pointer arithmetic. On platforms where size_t is narrower than int64_t, a large len-4 would truncate and could cause out-of-bounds reads. Reject a len-4 that exceeds SIZE_MAX and use a single `size_t inlen` for the payload length and CRC offset. Assisted-by: GitHub Copilot:claude-opus-4.8 --- lang/c/src/codec.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/lang/c/src/codec.c b/lang/c/src/codec.c index 2f3857957e3..5e8a3cda15e 100644 --- a/lang/c/src/codec.c +++ b/lang/c/src/codec.c @@ -198,7 +198,18 @@ static int decode_snappy(avro_codec_t c, void * data, int64_t len) return 1; } - if (snappy_uncompressed_length((const char*)data, len-4, &outlen) != SNAPPY_OK) { + /* Snappy's APIs take the compressed length as a size_t. On platforms + * where size_t is narrower than int64_t (e.g. 32-bit), a large len-4 + * would truncate during conversion and lead to out-of-bounds reads; + * reject it and use a single size_t for the payload length. */ + if ((uint64_t) (len - 4) > (uint64_t) SIZE_MAX) { + avro_set_error("Snappy compressed length %lld is out of range", + (long long) (len - 4)); + return 1; + } + size_t inlen = (size_t) (len - 4); + + if (snappy_uncompressed_length((const char*)data, inlen, &outlen) != SNAPPY_OK) { avro_set_error("Uncompressed length error in snappy"); return 1; } @@ -225,14 +236,14 @@ static int decode_snappy(avro_codec_t c, void * data, int64_t len) return 1; } - if (snappy_uncompress((const char*)data, len-4, (char*)c->block_data, &outlen) != SNAPPY_OK) + if (snappy_uncompress((const char*)data, inlen, (char*)c->block_data, &outlen) != SNAPPY_OK) { avro_set_error("Error uncompressing block with Snappy"); return 1; } crc = __bswap_32(crc32(0, (const Bytef *)c->block_data, outlen)); - if (memcmp(&crc, (char*)data+len-4, 4)) + if (memcmp(&crc, (char*)data+inlen, 4)) { avro_set_error("CRC32 check failure uncompressing block with Snappy"); return 1;