diff --git a/lang/csharp/src/apache/main/File/Codec.cs b/lang/csharp/src/apache/main/File/Codec.cs index 46191997a1d..06f1b626658 100644 --- a/lang/csharp/src/apache/main/File/Codec.cs +++ b/lang/csharp/src/apache/main/File/Codec.cs @@ -18,6 +18,7 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.IO; using System.Reflection; @@ -29,6 +30,98 @@ namespace Avro.File /// public abstract class Codec { + /// + /// Default upper bound, in bytes, on the size a single data-file block may + /// decompress to. A block with a very high compression ratio (or a malformed + /// block) can otherwise expand to far more memory than its compressed size. + /// Mirrors the Java SDK's decompression limit (AVRO-4247). Overridable with + /// the AVRO_MAX_DECOMPRESS_LENGTH environment variable. + /// + public static readonly long DefaultMaxDecompressLength = 200L * 1024 * 1024; // 200 MiB + + /// + /// Name of the environment variable used to override the default maximum + /// decompressed size of a single block. + /// + public const string MaxDecompressLengthEnvVar = "AVRO_MAX_DECOMPRESS_LENGTH"; + + /// + /// The maximum number of bytes a single block is allowed to decompress to. + /// + /// The configured limit, honoring the environment override. + public static long GetMaxDecompressLength() + { + var value = Environment.GetEnvironmentVariable(MaxDecompressLengthEnvVar); + if (value != null && long.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed) && parsed > 0) + { + return parsed; + } + + return DefaultMaxDecompressLength; + } + + /// + /// Throws if the given decompressed length exceeds the maximum allowed. + /// + /// The number of decompressed bytes. + /// The maximum number of decompressed bytes allowed. + public static void CheckDecompressLength(long length, long maxLength) + { + if (length > maxLength) + { + throw new AvroRuntimeException( + $"Decompressed block size {length} exceeds the maximum allowed of {maxLength} bytes. " + + $"For data-file reads, the {MaxDecompressLengthEnvVar} environment variable raises the limit."); + } + } + + /// + /// Copies a decompression stream to the destination, rejecting the block as + /// soon as its decompressed size would exceed so + /// an over-large (or malicious) block is not fully materialized in memory. + /// + /// The decompression stream to read from. + /// The stream to write the decompressed data to. + /// The maximum number of decompressed bytes allowed. + public static void CopyBounded(Stream source, Stream destination, long maxLength) + { + if (source == null) + { + throw new ArgumentNullException(nameof(source)); + } + + if (destination == null) + { + throw new ArgumentNullException(nameof(destination)); + } + + if (maxLength < 0) + { + throw new ArgumentOutOfRangeException(nameof(maxLength), "maxLength must not be negative."); + } + + byte[] buffer = new byte[81920]; + long total = 0; + int read; + while ((read = source.Read(buffer, 0, buffer.Length)) > 0) + { + // Pre-add bound check: total is always <= maxLength here and + // read > 0, so maxLength - total >= 0 and this cannot overflow. + // Rejecting before adding stops total from overflowing and + // wrapping past the limit for a very large maxLength. + if (read > maxLength - total) + { + throw new AvroRuntimeException( + $"Decompressed block size exceeds the maximum allowed of {maxLength} bytes " + + $"(at least {total} bytes already decompressed). " + + $"For data-file reads, the {MaxDecompressLengthEnvVar} environment variable raises the limit."); + } + + total += read; + destination.Write(buffer, 0, read); + } + } + /// /// Compress data using implemented codec. /// diff --git a/lang/csharp/src/apache/main/File/DataFileReader.cs b/lang/csharp/src/apache/main/File/DataFileReader.cs index dff13e05885..69e1deb3af8 100644 --- a/lang/csharp/src/apache/main/File/DataFileReader.cs +++ b/lang/csharp/src/apache/main/File/DataFileReader.cs @@ -335,6 +335,11 @@ public bool HasNext() { _currentBlock = NextRawBlock(_currentBlock); _currentBlock.Data = _codec.Decompress(_currentBlock.Data, (int)_blockSize); + // Guard against a block that decompresses to more than the + // allowed maximum (a decompression bomb). The built-in deflate + // codec is already bounded during decompression; this covers + // any codec that returns a fully decompressed buffer. + Codec.CheckDecompressLength(_currentBlock.Data.Length, Codec.GetMaxDecompressLength()); _datumDecoder = new BinaryDecoder(_currentBlock.GetDataAsStream()); } } @@ -343,7 +348,7 @@ public bool HasNext() catch (Exception e) { throw new AvroRuntimeException(string.Format(CultureInfo.InvariantCulture, - "Error fetching next object from block: {0}", e)); + "Error fetching next object from block: {0}", e.Message), e); } } diff --git a/lang/csharp/src/apache/main/File/DeflateCodec.cs b/lang/csharp/src/apache/main/File/DeflateCodec.cs index 0ce37adb092..0cedb64fe78 100644 --- a/lang/csharp/src/apache/main/File/DeflateCodec.cs +++ b/lang/csharp/src/apache/main/File/DeflateCodec.cs @@ -63,7 +63,10 @@ public override byte[] Decompress(byte[] compressedData, int length) { using (DeflateStream decompress = new DeflateStream(inStream, CompressionMode.Decompress)) { - decompress.CopyTo(outStream); + // Bound the decompressed size to guard against a block with a very + // high compression ratio expanding to far more memory than its + // compressed size. + CopyBounded(decompress, outStream, GetMaxDecompressLength()); } return outStream.ToArray(); } diff --git a/lang/csharp/src/apache/test/File/FileTests.cs b/lang/csharp/src/apache/test/File/FileTests.cs index abb3f9c6076..3ec74cb1a70 100644 --- a/lang/csharp/src/apache/test/File/FileTests.cs +++ b/lang/csharp/src/apache/test/File/FileTests.cs @@ -425,6 +425,112 @@ public void OpenAppendWriter_IncorrectOutStream_Throws() Assert.Throws(typeof(AvroRuntimeException), action); } + /// + /// A block with a very high compression ratio can expand to far more memory + /// than its compressed size; decompressing such a block must be rejected once + /// its decompressed size would exceed the configured maximum. + /// + [Test] + public void TestDeflateDecompressionLimit() + { + var codec = new DeflateCodec(); + byte[] big = new byte[128 * 1024]; // 128 KiB of zeros, compresses tiny + byte[] compressed = codec.Compress(big); + + var previous = Environment.GetEnvironmentVariable(Codec.MaxDecompressLengthEnvVar); + Environment.SetEnvironmentVariable(Codec.MaxDecompressLengthEnvVar, "65536"); // 64 KiB + try + { + Assert.Throws( + () => codec.Decompress(compressed, compressed.Length)); + } + finally + { + Environment.SetEnvironmentVariable(Codec.MaxDecompressLengthEnvVar, previous); + } + } + + [Test] + public void TestDeflateWithinDecompressionLimit() + { + var codec = new DeflateCodec(); + byte[] payload = System.Text.Encoding.UTF8.GetBytes("hello world"); + byte[] compressed = codec.Compress(payload); + + var previous = Environment.GetEnvironmentVariable(Codec.MaxDecompressLengthEnvVar); + Environment.SetEnvironmentVariable(Codec.MaxDecompressLengthEnvVar, "65536"); // 64 KiB + try + { + byte[] result = codec.Decompress(compressed, compressed.Length); + CollectionAssert.AreEqual(payload, result); + } + finally + { + Environment.SetEnvironmentVariable(Codec.MaxDecompressLengthEnvVar, previous); + } + } + + [Test] + public void TestCopyBoundedValidatesArguments() + { + using (var stream = new MemoryStream()) + { + Assert.Throws(() => Codec.CopyBounded(null, stream, 10)); + Assert.Throws(() => Codec.CopyBounded(stream, null, 10)); + Assert.Throws(() => Codec.CopyBounded(stream, stream, -1)); + } + } + + /// + /// The DataFileReader itself must reject a block whose decompressed size + /// exceeds the configured maximum. This covers the safeguard applied to + /// every codec that returns a fully materialized buffer (here the Null + /// codec, which performs no internally bounded decompression), not just a + /// direct call to a codec's Decompress method. + /// + [Test] + public void TestReaderRejectsOversizedBlock() + { + const string schemaStr = + "{\"type\":\"record\",\"name\":\"n\",\"fields\":[{\"name\":\"f1\",\"type\":\"string\"}]}"; + Schema schema = Schema.Parse(schemaStr); + var recordSchema = schema as RecordSchema; + + // A single record whose string field is larger than the limit below. + string big = new string('a', 128 * 1024); // 128 KiB + + MemoryStream outStream = new MemoryStream(); + using (var writer = DataFileWriter.OpenWriter( + new GenericWriter(schema), outStream, Codec.CreateCodec(Codec.Type.Null))) + { + writer.Append(mkRecord(new object[] { "f1", big }, recordSchema)); + } + + MemoryStream inStream = new MemoryStream(outStream.ToArray()); + + var previous = Environment.GetEnvironmentVariable(Codec.MaxDecompressLengthEnvVar); + Environment.SetEnvironmentVariable(Codec.MaxDecompressLengthEnvVar, "65536"); // 64 KiB + try + { + Assert.Throws(() => + { + using (var reader = DataFileReader.OpenReader(inStream, schema)) + { + // Enumerating forces the block to be read and + // decompressed, which is where the limit is enforced. + foreach (var rec in reader.NextEntries) + { + Assert.NotNull(rec); + } + } + }); + } + finally + { + Environment.SetEnvironmentVariable(Codec.MaxDecompressLengthEnvVar, previous); + } + } + /// /// This test is a single test case of /// but introduces a