Skip to content

Commit 673f0ae

Browse files
author
Grok Compression
committed
fetch: make HTTP/2 multiplexing opt-in via http2_multiplex flag
HTTP/2 multiplexing (CURLPIPE_MULTIPLEX + HTTP_VERSION_2TLS + PIPEWAIT) caused a 10-15% regression on LRCP files with CloudFerro. Move it behind grk_stream_params::http2_multiplex (default false) so callers can enable it when beneficial.
1 parent 98655c0 commit 673f0ae

4 files changed

Lines changed: 14 additions & 3 deletions

File tree

src/lib/core/grok.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -628,6 +628,9 @@ typedef struct _grk_stream_params
628628
/* 13 Fetch concurrency (0 = use default of 30) */
629629
uint32_t fetch_batch_size; /* max concurrent HTTP range requests */
630630

631+
/* 14 HTTP/2 multiplexing (0 = disabled, 1 = enabled) */
632+
bool http2_multiplex;
633+
631634
} grk_stream_params;
632635

633636
/**

src/lib/core/stream/StreamGenerator.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,7 @@ IStream* StreamGenerator::createCurlFetchStream(void)
161161
auth.max_retry_ = streamParams_.max_retry;
162162
auth.retry_delay_ = streamParams_.retry_delay;
163163
auth.fetch_batch_size_ = streamParams_.fetch_batch_size;
164+
auth.http2_multiplex_ = streamParams_.http2_multiplex;
164165
grklog.debug("StreamGenerator: s3_allow_insecure: streamParams=%d, auth=%d",
165166
(int)streamParams_.s3_allow_insecure, (int)auth.s3_allow_insecure_);
166167
std::string_view file{streamParams_.file};

src/lib/core/stream/fetchers/CurlFetcher.cpp

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,6 @@ CurlFetcher::CurlFetcher(void) : tileWriteCallback_(tileWriteCallback)
7676
if(!multi_handle_)
7777
throw std::runtime_error("Failed to initialize CURL multi handle");
7878
curl_multi_setopt(multi_handle_, CURLMOPT_MAX_TOTAL_CONNECTIONS, 100L);
79-
curl_multi_setopt(multi_handle_, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
8079
fetchThread_ = std::thread(&CurlFetcher::fetchWorker, this);
8180
}
8281

@@ -120,6 +119,8 @@ void CurlFetcher::init(const std::string& path, const FetchAuth& auth)
120119
retryDelayMs_ = auth_.retry_delay_ * 1000;
121120
if(auth_.fetch_batch_size_ > 0)
122121
batchSize_ = auth_.fetch_batch_size_;
122+
if(auth_.http2_multiplex_)
123+
curl_multi_setopt(multi_handle_, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
123124
parse(path);
124125
fetch_total_size();
125126
}
@@ -569,8 +570,11 @@ CURL* CurlFetcher::configureHandle(uint64_t offset, uint64_t end, FetchResult& r
569570
curl_easy_setopt(curl, CURLOPT_URL, url_.c_str());
570571
curl_initiate_retry(curl);
571572
curl_easy_setopt(curl, CURLOPT_VERBOSE, 0L);
572-
curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS);
573-
curl_easy_setopt(curl, CURLOPT_PIPEWAIT, 1L);
573+
if(auth_.http2_multiplex_)
574+
{
575+
curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS);
576+
curl_easy_setopt(curl, CURLOPT_PIPEWAIT, 1L);
577+
}
574578
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, callback);
575579
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &result);
576580
curl_easy_setopt(curl, CURLOPT_PRIVATE, &result);

src/lib/core/stream/fetchers/FetchCommon.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,9 @@ struct FetchAuth
8282
// Fetch concurrency (0 = use default of 30)
8383
uint32_t fetch_batch_size_ = 0;
8484

85+
// HTTP/2 multiplexing (disabled by default)
86+
bool http2_multiplex_ = false;
87+
8588
FetchAuth() = default;
8689
FetchAuth(const std::string& u, const std::string& p, const std::string& t,
8790
const std::vector<std::string>& h = {}, const std::string& r = "",

0 commit comments

Comments
 (0)