diff --git a/lang/js/lib/schemas.js b/lang/js/lib/schemas.js index d99eedb8a10..ba0d5f8becc 100644 --- a/lang/js/lib/schemas.js +++ b/lang/js/lib/schemas.js @@ -65,6 +65,26 @@ var PATH = []; // Currently active logical type, used for name redirection. var LOGICAL_TYPE = null; +// Collection allocation limits, guarding against a block-count DoS. +// +// An `array` or `map` block is encoded as an item count followed by that many +// items. A malicious or truncated input can declare a very large count while +// carrying little or no data. Elements that encode to zero bytes (e.g. `null`) +// consume no input, so the block count cannot be bounded by the bytes actually +// remaining in the buffer alone; such a block is capped by item count instead. +// Both limits can be overridden (to the same value) via the +// `AVRO_MAX_COLLECTION_ITEMS` environment variable. +var MAX_COLLECTION_ITEMS = 10000000; // Zero-byte element cap. +var MAX_COLLECTION_STRUCTURAL = 2147483639; // Integer.MAX_VALUE - 8. +if (typeof process !== 'undefined' && process.env && + process.env.AVRO_MAX_COLLECTION_ITEMS) { + var envItems = parseInt(process.env.AVRO_MAX_COLLECTION_ITEMS, 10); + if (envItems >= 0) { + MAX_COLLECTION_ITEMS = envItems; + MAX_COLLECTION_STRUCTURAL = envItems; + } +} + /** * Schema parsing entry point. @@ -782,7 +802,12 @@ UnionType.prototype._read = function (tap) { }; UnionType.prototype._skip = function (tap) { - this._types[tap.readLong()]._skip(tap); + var index = tap.readLong(); + var type = this._types[index]; + if (type === undefined) { + throw new Error(f('invalid union index: %s', index)); + } + type._skip(tap); }; UnionType.prototype._write = function (tap, val) { @@ -1149,9 +1174,15 @@ MapType.prototype._check = function (val, cb) { MapType.prototype._read = function (tap) { var values = this._values; + var minBytes = this._valuesMinBytes !== undefined ? + this._valuesMinBytes : + 1 + getMinBytes(values); // Each entry carries a >=1-byte key. var val = {}; + var total = 0; var n; while ((n = readArraySize(tap))) { + total += n; + checkCollectionBlock(tap, n, minBytes, total); while (n--) { var key = tap.readString(); val[key] = values._read(tap); @@ -1162,12 +1193,37 @@ MapType.prototype._read = function (tap) { MapType.prototype._skip = function (tap) { var values = this._values; + var minBytes = 1 + getMinBytes(values); // Each entry carries a >=1-byte key. + var total = 0; var len, n; while ((n = tap.readLong())) { if (n < 0) { + // Sized block: bound the item count too, so the cap cannot be bypassed by + // encoding the block with a negative (byte-sized) count. + n = -n; len = tap.readLong(); + if (len < 0) { + // A negative byte-size would move the read position backwards, which + // Tap.isValid() (pos <= buf.length) would not catch, bypassing + // truncation detection. + throw new Error('negative collection block size'); + } + if (len > tap.buf.length - tap.pos) { + // The block claims more bytes than remain; skipping it would jump past + // the buffer end and misalign subsequent decoding. + throw new Error('collection block size exceeds remaining buffer'); + } + if (minBytes > 0 && n > len / minBytes) { + // The declared byte-size is too small to hold n entries at their minimum + // on-wire size, so skipping len bytes would land mid-entry. + throw new Error('collection block size inconsistent with item count'); + } + total += n; + checkCollectionBlock(tap, n, minBytes, total); tap.pos += len; } else { + total += n; + checkCollectionBlock(tap, n, minBytes, total); while (n--) { tap.skipString(); values._skip(tap); @@ -1203,6 +1259,8 @@ MapType.prototype._match = function () { MapType.prototype._updateResolver = function (resolver, type, opts) { if (type instanceof MapType) { resolver._values = this._values.createResolver(type._values, opts); + // Bound the block count using the writer's entry size (see ArrayType). + resolver._valuesMinBytes = 1 + getMinBytes(type._values); resolver._read = this._read; } }; @@ -1288,13 +1346,19 @@ ArrayType.prototype._check = function (val, cb) { ArrayType.prototype._read = function (tap) { var items = this._items; + var minBytes = this._itemsMinBytes !== undefined ? + this._itemsMinBytes : + getMinBytes(items); var val = []; + var total = 0; var n; while ((n = tap.readLong())) { if (n < 0) { n = -n; tap.skipLong(); // Skip size. } + total += n; + checkCollectionBlock(tap, n, minBytes, total); while (n--) { val.push(items._read(tap)); } @@ -1303,14 +1367,40 @@ ArrayType.prototype._read = function (tap) { }; ArrayType.prototype._skip = function (tap) { + var items = this._items; + var minBytes = getMinBytes(items); + var total = 0; var len, n; while ((n = tap.readLong())) { if (n < 0) { + // Sized block: bound the item count too, so the cap cannot be bypassed by + // encoding the block with a negative (byte-sized) count. + n = -n; len = tap.readLong(); + if (len < 0) { + // A negative byte-size would move the read position backwards, which + // Tap.isValid() (pos <= buf.length) would not catch, bypassing + // truncation detection. + throw new Error('negative collection block size'); + } + if (len > tap.buf.length - tap.pos) { + // The block claims more bytes than remain; skipping it would jump past + // the buffer end and misalign subsequent decoding. + throw new Error('collection block size exceeds remaining buffer'); + } + if (minBytes > 0 && n > len / minBytes) { + // The declared byte-size is too small to hold n items at their minimum + // on-wire size, so skipping len bytes would land mid-entry. + throw new Error('collection block size inconsistent with item count'); + } + total += n; + checkCollectionBlock(tap, n, minBytes, total); tap.pos += len; } else { + total += n; + checkCollectionBlock(tap, n, minBytes, total); while (n--) { - this._items._skip(tap); + items._skip(tap); } } } @@ -1354,6 +1444,9 @@ ArrayType.prototype._match = function (tap1, tap2) { ArrayType.prototype._updateResolver = function (resolver, type, opts) { if (type instanceof ArrayType) { resolver._items = this._items.createResolver(type._items, opts); + // Bound the block count using the writer's element size, which is what is + // actually on the wire (the reader/resolved type may be larger). + resolver._itemsMinBytes = getMinBytes(type._items); resolver._read = this._read; } }; @@ -1988,10 +2081,12 @@ function Resolver(readerType) { // Add all fields here so that all resolvers share the same hidden class. this._readerType = readerType; this._items = null; + this._itemsMinBytes = undefined; this._read = null; this._size = 0; this._symbols = null; this._values = null; + this._valuesMinBytes = undefined; } Resolver.prototype.inspect = function () { return ''; }; @@ -2153,6 +2248,90 @@ function readArraySize(tap) { return n; } +/** + * Minimum number of bytes a value of the given type can occupy on the wire. + * + * Used to bound collection block counts against the bytes actually remaining. + * Memoized on the type; recursive records are handled with a `seen` guard that + * returns 0 when a cycle is detected. That 0 is a deliberately conservative + * lower bound: for a schema whose true minimum can't be determined without + * unbounded recursion, under-estimating can only make the bytes-remaining check + * more permissive (it never rejects otherwise-valid data), so it is safe for + * that check even though it may be smaller than the true minimum for a union + * whose only small branch is a recursive record. + */ +function getMinBytes(type, seen) { + if (type._minBytes !== undefined) { + return type._minBytes; + } + var mb; + if (type instanceof NullType) { + mb = 0; + } else if (type instanceof FixedType) { + mb = type._size; + } else if (type instanceof FloatType) { + mb = 4; + } else if (type instanceof DoubleType) { + mb = 8; + } else if (type instanceof LogicalType) { + mb = getMinBytes(type._underlyingType, seen); + } else if (type instanceof UnionType) { + // A 1-byte branch index plus the smallest branch encoding. A union with a + // zero-byte branch (e.g. null) still occupies the index byte, so the + // minimum is 1; a union without one (e.g. ['int','long']) is at least 2. + var branches = type._types; + var branchMin = branches.length ? Infinity : 0; + for (var k = 0, bl = branches.length; k < bl; k++) { + branchMin = Math.min(branchMin, getMinBytes(branches[k], seen)); + if (branchMin === 0) { + break; + } + } + mb = 1 + branchMin; + } else if (type instanceof RecordType) { + seen = seen || []; + if (~seen.indexOf(type)) { + return 0; // Cycle; see note above. + } + seen.push(type); + mb = 0; + var fields = type._fields; + for (var i = 0, l = fields.length; i < l; i++) { + mb += getMinBytes(fields[i]._type, seen); + } + seen.pop(); + } else { + // Booleans, ints, longs, strings, bytes, enums (index), unions (index), + // and arrays/maps (empty block terminator) all occupy at least one byte. + mb = 1; + } + type._minBytes = mb; + return mb; +} + +/** + * Reject a collection block whose declared item count cannot be backed by the + * input, guarding against unbounded allocation from a tiny payload. + * + * `total` is the cumulative item count across all blocks read so far (including + * the current one). Elements with a positive on-wire minimum are bounded by the + * bytes remaining in the buffer; zero-byte elements are bounded by the item + * cap; and every collection is bounded by the structural cap. + */ +function checkCollectionBlock(tap, n, minBytes, total) { + if (total > MAX_COLLECTION_STRUCTURAL) { + throw new Error('collection size exceeds maximum allowed'); + } + if (minBytes > 0) { + // Divide (rather than multiply) to avoid any overflow on large counts. + if (n > (tap.buf.length - tap.pos) / minBytes) { + throw new Error('collection size exceeds remaining buffer'); + } + } else if (total > MAX_COLLECTION_ITEMS) { + throw new Error('collection of zero-byte items exceeds maximum allowed'); + } +} + /** * Correctly stringify an object which contains types. * diff --git a/lang/js/lib/utils.js b/lang/js/lib/utils.js index 81d16c4fdde..e655798f0ab 100644 --- a/lang/js/lib/utils.js +++ b/lang/js/lib/utils.js @@ -335,10 +335,18 @@ Tap.prototype.readInt = Tap.prototype.readLong = function () { // Switch to float arithmetic, otherwise we might overflow. f = n; fk = 268435456; // 2 ** 28. + // A 64-bit value uses at most 10 bytes; the integer loop above consumed 4, + // so the float loop may read at most 6 more. Reject an overlong varint + // rather than reading an unbounded continuation chain. + var m = 0; do { + if (m === 6) { + throw new Error('overlong varint'); + } b = buf[this.pos++]; f += (b & 0x7f) * fk; fk *= 128; + m++; } while (b & 0x80); return (f % 2 ? -(f + 1) : f) / 2; } @@ -348,7 +356,13 @@ Tap.prototype.readInt = Tap.prototype.readLong = function () { Tap.prototype.skipInt = Tap.prototype.skipLong = function () { var buf = this.buf; - while (buf[this.pos++] & 0x80) {} + var m = 0; + while (buf[this.pos++] & 0x80) { + // At most 10 bytes for a 64-bit varint; reject an overlong encoding. + if (++m === 10) { + throw new Error('overlong varint'); + } + } }; Tap.prototype.writeInt = Tap.prototype.writeLong = function (n) { diff --git a/lang/js/test/test_schemas.js b/lang/js/test/test_schemas.js index d8bbcd116b7..0f81cd27828 100644 --- a/lang/js/test/test_schemas.js +++ b/lang/js/test/test_schemas.js @@ -419,6 +419,15 @@ describe('types', function () { assert.throws(function () { type.fromBuffer(buf); }); }); + it('skip invalid index', function () { + var type = new types.UnionType(['null', 'int']); + var buf = Buffer.alloc(16); + var tap = new Tap(buf); + tap.writeLong(5); // out of range for a 2-branch union + tap.pos = 0; + assert.throws(function () { type._skip(tap); }, /invalid union index/); + }); + it('non wrapped write', function () { var type = new types.UnionType(['null', 'int']); assert.throws(function () { @@ -939,6 +948,86 @@ describe('types', function () { assert.strictEqual(t.getName(), undefined); }); + it('rejects a huge block count', function () { + var t = new types.MapType({type: 'map', values: 'long'}); + var buf = Buffer.from([0x80, 0x88, 0xde, 0xbe, 0x01, 0x00]); + assert.throws(function () { t.fromBuffer(buf); }, /collection/); + }); + + it('reads a small map', function () { + var t = new types.MapType({type: 'map', values: 'int'}); + var buf = t.toBuffer({a: 1, b: 2}); + assert.deepEqual(t.fromBuffer(buf), {a: 1, b: 2}); + }); + + it('bounds a huge block count while skipping', function () { + var v1 = createType({ + name: 'Foo', + type: 'record', + fields: [ + {name: 'map', type: {type: 'map', values: 'null'}}, + {name: 'val', type: 'int'} + ] + }); + var v2 = createType({ + name: 'Foo', + type: 'record', + fields: [{name: 'val', type: 'int'}] + }); + var buf = Buffer.from([0x80, 0x88, 0xde, 0xbe, 0x01, 0x00, 6]); + var resolver = v2.createResolver(v1); + assert.throws(function () { v2.fromBuffer(buf, resolver); }, /collection/); + }); + + it('bounds a huge negative (sized) block count while skipping', function () { + var v1 = createType({ + name: 'Foo', + type: 'record', + fields: [ + {name: 'map', type: {type: 'map', values: 'null'}}, + {name: 'val', type: 'int'} + ] + }); + var v2 = createType({ + name: 'Foo', + type: 'record', + fields: [{name: 'val', type: 'int'}] + }); + // Block count -200,000,000, a block byte-size of 0, then terminator + val. + var buf = Buffer.from([0xff, 0x87, 0xde, 0xbe, 0x01, 0x00, 0x00, 6]); + var resolver = v2.createResolver(v1); + assert.throws(function () { v2.fromBuffer(buf, resolver); }, /collection/); + }); + + it('rejects a negative block byte-size while skipping', function () { + var v1 = createType({ + name: 'Foo', + type: 'record', + fields: [ + {name: 'map', type: {type: 'map', values: 'int'}}, + {name: 'val', type: 'int'} + ] + }); + var v2 = createType({ + name: 'Foo', + type: 'record', + fields: [{name: 'val', type: 'int'}] + }); + // Block count -1 followed by a negative byte-size (zig-zag 1 -> 0x01). + var buf = Buffer.from([0x01, 0x01, 0x00, 6]); + var resolver = v2.createResolver(v1); + assert.throws(function () { v2.fromBuffer(buf, resolver); }, /negative/); + }); + + it('reads a small map under resolution', function () { + var t1 = new types.MapType({type: 'map', values: 'int'}); + var t2 = createType({type: 'map', values: ['null', 'int']}); + var buf = t1.toBuffer({a: 1, b: 2}); + var resolver = t2.createResolver(t1); + var result = t2.fromBuffer(buf, resolver); + assert.deepEqual(Object.keys(result).sort(), ['a', 'b']); + }); + }); describe('ArrayType', function () { @@ -971,6 +1060,93 @@ describe('types', function () { assert.deepEqual(t.fromBuffer(buf), [1]); }); + it('rejects a huge zero-byte block count', function () { + // 6-byte payload declaring a block count of 200,000,000 nulls. + var t = new types.ArrayType({type: 'array', items: 'null'}); + var buf = Buffer.from([0x80, 0x88, 0xde, 0xbe, 0x01, 0x00]); + assert.throws(function () { t.fromBuffer(buf); }, /collection/); + }); + + it('reads a small zero-byte collection', function () { + var t = new types.ArrayType({type: 'array', items: 'null'}); + var buf = Buffer.from([6, 0]); // Block count 3, then terminator. + assert.deepEqual(t.fromBuffer(buf), [null, null, null]); + }); + + it('rejects an INT64_MIN block count', function () { + // INT64_MIN zig-zags to the 10-byte varint below (then a block byte-size). + // Negating it yields ~2^63, which the structural cap rejects. + var t = new types.ArrayType({type: 'array', items: 'null'}); + var buf = Buffer.from( + [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01, 0x00]); + assert.throws(function () { t.fromBuffer(buf); }, /collection/); + }); + + it('rejects a block count above the remaining bytes', function () { + var t = new types.ArrayType({type: 'array', items: 'long'}); + var buf = Buffer.from([0x80, 0x88, 0xde, 0xbe, 0x01, 0x00]); + assert.throws(function () { t.fromBuffer(buf); }, /collection/); + }); + + it('bounds a huge zero-byte block count while skipping', function () { + var v1 = createType({ + name: 'Foo', + type: 'record', + fields: [ + {name: 'array', type: {type: 'array', items: 'null'}}, + {name: 'val', type: 'int'} + ] + }); + var v2 = createType({ + name: 'Foo', + type: 'record', + fields: [{name: 'val', type: 'int'}] + }); + var buf = Buffer.from([0x80, 0x88, 0xde, 0xbe, 0x01, 0x00, 6]); + var resolver = v2.createResolver(v1); + assert.throws(function () { v2.fromBuffer(buf, resolver); }, /collection/); + }); + + it('bounds a huge negative (sized) block count while skipping', function () { + // A negative block count carries a byte-size and is skipped in one step; + // the item cap must still apply so it cannot be used to bypass the bound. + var v1 = createType({ + name: 'Foo', + type: 'record', + fields: [ + {name: 'array', type: {type: 'array', items: 'null'}}, + {name: 'val', type: 'int'} + ] + }); + var v2 = createType({ + name: 'Foo', + type: 'record', + fields: [{name: 'val', type: 'int'}] + }); + // Block count -200,000,000 (zig-zag 0xFF..), a block byte-size of 0, then + // the terminator and val. + var buf = Buffer.from( + [0xff, 0x87, 0xde, 0xbe, 0x01, 0x00, 0x00, 6]); + var resolver = v2.createResolver(v1); + assert.throws(function () { v2.fromBuffer(buf, resolver); }, /collection/); + }); + + it('bounds a huge block count under resolution', function () { + var t1 = new types.ArrayType({type: 'array', items: 'null'}); + var t2 = createType({type: 'array', items: ['null', 'long']}); + var buf = Buffer.from([0x80, 0x88, 0xde, 0xbe, 0x01, 0x00]); + var resolver = t2.createResolver(t1); + assert.throws(function () { t2.fromBuffer(buf, resolver); }, /collection/); + }); + + it('reads a small collection under resolution', function () { + var t1 = new types.ArrayType({type: 'array', items: 'null'}); + var t2 = createType({type: 'array', items: ['null', 'long']}); + var buf = t1.toBuffer([null, null, null, null, null]); + var resolver = t2.createResolver(t1); + assert.equal(t2.fromBuffer(buf, resolver).length, 5); + }); + it('skip', function () { var v1 = createType({ name: 'Foo', diff --git a/lang/js/test/test_utils.js b/lang/js/test/test_utils.js index ca72891e71a..adc66aae179 100644 --- a/lang/js/test/test_utils.js +++ b/lang/js/test/test_utils.js @@ -166,6 +166,24 @@ describe('utils', function () { }); + it('read rejects overlong varint', function () { + // 10 continuation bytes then a terminator: a 64-bit value uses at most + // 10 bytes, so this overlong encoding must be rejected. + var bytes = []; + for (var i = 0; i < 10; i++) { bytes.push(0x80); } + bytes.push(0x01); + var buf = Buffer.from(bytes); + assert.throws(function () { (new Tap(buf)).readLong(); }, /overlong varint/); + }); + + it('skip rejects overlong varint', function () { + var bytes = []; + for (var i = 0; i < 10; i++) { bytes.push(0x80); } + bytes.push(0x01); + var buf = Buffer.from(bytes); + assert.throws(function () { (new Tap(buf)).skipLong(); }, /overlong varint/); + }); + }); describe('boolean', function () {