Cap uncompressed length in TLS Certificate Compression (#13117)

* Cap uncompressed length in TLS Certificate Compression

* Set a nullptr

* Update the comment

Author: maskit

src/iocore/net/TLSCertCompression.h

@@ -27,6 +27,12 @@
 #include <vector>
 #include <string>
 
+// RFC 8879 uses uint24 for uncompressed_length, allowing up to ~16 MB.
+// Real certificate chains rarely exceed 10-30 KB even with large RSA
+// keys and multiple intermediates — 128 KB gives ample headroom while
+// preventing excessive memory allocation from a malicious peer.
+constexpr size_t MAX_CERT_UNCOMPRESSED_LEN = 128 * 1024;
+
 /**
  * Common function to set certificate compression preference
  *

src/iocore/net/TLSCertCompression_brotli.cc

@@ -22,6 +22,7 @@
  */
 
 #include "TLSCertCompression_brotli.h"
+#include "TLSCertCompression.h"
 #include "SSLStats.h"
 #include <openssl/ssl.h>
 #include <brotli/decode.h>
@@ -55,6 +56,12 @@ compression_func_brotli(SSL * /* ssl */, CBB *out, const uint8_t *in, size_t in_
 int
 decompression_func_brotli(SSL * /* ssl */, CRYPTO_BUFFER **out, size_t uncompressed_len, const uint8_t *in, size_t in_len)
 {
+  if (uncompressed_len > MAX_CERT_UNCOMPRESSED_LEN) {
+    *out = nullptr;
+    Metrics::Counter::increment(ssl_rsb.cert_decompress_brotli_failure);
+    return 0;
+  }
+
   uint8_t *buf;
 
   *out = CRYPTO_BUFFER_alloc(&buf, uncompressed_len);

src/iocore/net/TLSCertCompression_zlib.cc

@@ -22,6 +22,7 @@
  */
 
 #include "TLSCertCompression_zlib.h"
+#include "TLSCertCompression.h"
 #include "SSLStats.h"
 #include <openssl/ssl.h>
 #include <zlib.h>
@@ -53,6 +54,12 @@ compression_func_zlib(SSL * /* ssl */, CBB *out, const uint8_t *in, size_t in_le
 int
 decompression_func_zlib(SSL * /* ssl */, CRYPTO_BUFFER **out, size_t uncompressed_len, const uint8_t *in, size_t in_len)
 {
+  if (uncompressed_len > MAX_CERT_UNCOMPRESSED_LEN) {
+    *out = nullptr;
+    Metrics::Counter::increment(ssl_rsb.cert_decompress_zlib_failure);
+    return 0;
+  }
+
   uint8_t *buf;
 
   *out = CRYPTO_BUFFER_alloc(&buf, uncompressed_len);

src/iocore/net/TLSCertCompression_zstd.cc

@@ -22,6 +22,7 @@
  */
 
 #include "TLSCertCompression_zstd.h"
+#include "TLSCertCompression.h"
 #include "SSLStats.h"
 #include <openssl/ssl.h>
 #include <zstd.h>
@@ -60,6 +61,12 @@ compression_func_zstd(SSL * /* ssl */, CBB *out, const uint8_t *in, size_t in_le
 int
 decompression_func_zstd(SSL * /* ssl */, CRYPTO_BUFFER **out, size_t uncompressed_len, const uint8_t *in, size_t in_len)
 {
+  if (uncompressed_len > MAX_CERT_UNCOMPRESSED_LEN) {
+    *out = nullptr;
+    Metrics::Counter::increment(ssl_rsb.cert_decompress_zstd_failure);
+    return 0;
+  }
+
   uint8_t *buf;
 
   *out = CRYPTO_BUFFER_alloc(&buf, uncompressed_len);