crypto: guard against size_t overflow on experimental 32-bit arch

Refs: https://hackerone.com/reports/3655230

Signed-off-by: Filip Skokan <panva.ip@gmail.com>

Author: panva

src/crypto/crypto_turboshake.cc

@@ -589,7 +589,6 @@ bool KangarooTwelveTraits::DeriveBits(Environment* env,
                                       CryptoJobMode mode,
                                       CryptoErrorStore* errors) {
   CHECK_GT(params.output_length, 0);
-  char* buf = MallocOpenSSL<char>(params.output_length);
 
   const uint8_t* input = reinterpret_cast<const uint8_t*>(params.data.data());
   size_t input_len = params.data.size();
@@ -598,6 +597,18 @@ bool KangarooTwelveTraits::DeriveBits(Environment* env,
       reinterpret_cast<const uint8_t*>(params.customization.data());
   size_t custom_len = params.customization.size();
 
+  // Guard against size_t overflow in KangarooTwelve's s_len computation:
+  //   s_len = msg_len + custom_len + LengthEncode(custom_len).size()
+  // LengthEncode produces at most sizeof(size_t) + 1 bytes.
+  static constexpr size_t kMaxLengthEncodeSize = sizeof(size_t) + 1;
+  if (input_len > SIZE_MAX - custom_len ||
+      input_len + custom_len > SIZE_MAX - kMaxLengthEncodeSize) {
+    errors->Insert(NodeCryptoError::DERIVING_BITS_FAILED);
+    return false;
+  }
+
+  char* buf = MallocOpenSSL<char>(params.output_length);
+
   switch (params.variant) {
     case KangarooTwelveVariant::KT128:
       KT128(input,

test/pummel/test-webcrypto-kangarootwelve-32bit-overflow.js

@@ -0,0 +1,41 @@
+'use strict';
+const common = require('../common');
+
+if (!common.hasCrypto)
+  common.skip('missing crypto');
+
+// KangarooTwelve: data + customization size_t overflow on 32-bit platforms.
+// On 32-bit, msg_len + custom_len + LengthEncode(custom_len).size() can wrap
+// size_t, causing an undersized allocation and heap buffer overflow.
+// This test verifies the guard rejects such inputs.
+// When kMaxLength < 2^32 two max-sized buffers can overflow a 32-bit size_t.
+
+const { kMaxLength } = require('buffer');
+
+if (kMaxLength >= 2 ** 32)
+  common.skip('only relevant when kMaxLength < 2^32');
+
+const assert = require('assert');
+const { subtle } = globalThis.crypto;
+
+let data, customization;
+try {
+  data = new Uint8Array(kMaxLength);
+  customization = new Uint8Array(kMaxLength);
+} catch {
+  common.skip('insufficient memory to allocate test buffers');
+}
+
+(async () => {
+  await assert.rejects(
+    subtle.digest(
+      { name: 'KT128', outputLength: 256, customization },
+      data),
+    { name: 'OperationError' });
+
+  await assert.rejects(
+    subtle.digest(
+      { name: 'KT256', outputLength: 512, customization },
+      data),
+    { name: 'OperationError' });
+})().then(common.mustCall());