Skip to content

Commit 04c9b39

Browse files
mpeddada1mpeddada1
authored
impl(storage): clean up remaining client_options code (googleapis#15868)
* impl(storage): clean up remaining client_options code
1 parent 3702697 commit 04c9b39

17 files changed

Lines changed: 526 additions & 651 deletions

‎google/cloud/storage/client.cc‎

Lines changed: 227 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,19 +13,35 @@
1313
// limitations under the License.
1414

1515
#include "google/cloud/storage/client.h"
16+
#include "google/cloud/storage/idempotency_policy.h"
1617
#include "google/cloud/storage/internal/base64.h"
1718
#include "google/cloud/storage/internal/connection_factory.h"
19+
#include "google/cloud/storage/internal/unified_rest_credentials.h"
20+
#include "google/cloud/storage/oauth2/credentials.h"
21+
#include "google/cloud/storage/oauth2/google_credentials.h"
1822
#include "google/cloud/storage/oauth2/service_account_credentials.h"
23+
#include "google/cloud/storage/options.h"
1924
#include "google/cloud/internal/absl_str_cat_quiet.h"
25+
#include "google/cloud/internal/absl_str_join_quiet.h"
2026
#include "google/cloud/internal/curl_handle.h"
2127
#include "google/cloud/internal/curl_options.h"
2228
#include "google/cloud/internal/filesystem.h"
29+
#include "google/cloud/internal/getenv.h"
2330
#include "google/cloud/internal/make_status.h"
2431
#include "google/cloud/internal/opentelemetry.h"
32+
#include "google/cloud/internal/populate_common_options.h"
33+
#include "google/cloud/internal/rest_options.h"
34+
#include "google/cloud/internal/rest_response.h"
35+
#include "google/cloud/internal/service_endpoint.h"
2536
#include "google/cloud/log.h"
37+
#include "google/cloud/opentelemetry_options.h"
38+
#include "google/cloud/universe_domain_options.h"
2639
#include "absl/strings/str_split.h"
40+
#include <cstdlib>
2741
#include <fstream>
2842
#include <memory>
43+
#include <set>
44+
#include <sstream>
2945
#include <string>
3046
#include <thread>
3147
#include <utility>
@@ -387,8 +403,219 @@ std::string Client::EndpointAuthority() const {
387403
return std::string(endpoint_authority);
388404
}
389405

406+
// This magic number was obtained by experimentation summarized in #2657
407+
#ifndef GOOGLE_CLOUD_CPP_STORAGE_DEFAULT_UPLOAD_BUFFER_SIZE
408+
#define GOOGLE_CLOUD_CPP_STORAGE_DEFAULT_UPLOAD_BUFFER_SIZE (8 * 1024 * 1024)
409+
#endif // GOOGLE_CLOUD_CPP_STORAGE_DEFAULT_UPLOAD_BUFFER_SIZE
410+
411+
// This magic number was obtained by experimentation summarized in #2657
412+
#ifndef GOOGLE_CLOUD_CPP_STORAGE_DEFAULT_DOWNLOAD_BUFFER_SIZE
413+
#define GOOGLE_CLOUD_CPP_STORAGE_DEFAULT_DOWNLOAD_BUFFER_SIZE \
414+
(3 * 1024 * 1024 / 2)
415+
#endif // GOOGLE_CLOUD_CPP_STORAGE_DEFAULT_DOWNLOAD_BUFFER_SIZE
416+
417+
// This is a result of experiments performed in #2657.
418+
#ifndef GOOGLE_CLOUD_CPP_STORAGE_DEFAULT_MAXIMUM_SIMPLE_UPLOAD_SIZE
419+
#define GOOGLE_CLOUD_CPP_STORAGE_DEFAULT_MAXIMUM_SIMPLE_UPLOAD_SIZE \
420+
(20 * 1024 * 1024L)
421+
#endif // GOOGLE_CLOUD_CPP_STORAGE_DEFAULT_MAXIMUM_SIMPLE_UPLOAD_SIZE
422+
423+
#ifndef GOOGLE_CLOUD_CPP_STORAGE_DEFAULT_DOWNLOAD_STALL_TIMEOUT
424+
#define GOOGLE_CLOUD_CPP_STORAGE_DEFAULT_DOWNLOAD_STALL_TIMEOUT 120
425+
#endif // GOOGLE_CLOUD_CPP_STORAGE_DEFAULT_DOWNLOAD_STALL_TIMEOUT
426+
427+
// Define the defaults using a pre-processor macro, this allows the application
428+
// developers to change the defaults for their application by compiling with
429+
// different values.
430+
#ifndef STORAGE_CLIENT_DEFAULT_MAXIMUM_RETRY_PERIOD
431+
#define STORAGE_CLIENT_DEFAULT_MAXIMUM_RETRY_PERIOD std::chrono::minutes(15)
432+
#endif // STORAGE_CLIENT_DEFAULT_MAXIMUM_RETRY_PERIOD
433+
434+
#ifndef STORAGE_CLIENT_DEFAULT_INITIAL_BACKOFF_DELAY
435+
#define STORAGE_CLIENT_DEFAULT_INITIAL_BACKOFF_DELAY std::chrono::seconds(1)
436+
#endif // STORAGE_CLIENT_DEFAULT_INITIAL_BACKOFF_DELAY
437+
438+
#ifndef STORAGE_CLIENT_DEFAULT_MAXIMUM_BACKOFF_DELAY
439+
#define STORAGE_CLIENT_DEFAULT_MAXIMUM_BACKOFF_DELAY std::chrono::minutes(5)
440+
#endif // STORAGE_CLIENT_DEFAULT_MAXIMUM_BACKOFF_DELAY
441+
442+
#ifndef STORAGE_CLIENT_DEFAULT_BACKOFF_SCALING
443+
#define STORAGE_CLIENT_DEFAULT_BACKOFF_SCALING 2.0
444+
#endif // STORAGE_CLIENT_DEFAULT_BACKOFF_SCALING
445+
446+
namespace {
447+
448+
using ::google::cloud::internal::GetEnv;
449+
450+
absl::optional<std::string> GetEmulator() {
451+
auto emulator = GetEnv("CLOUD_STORAGE_EMULATOR_ENDPOINT");
452+
if (emulator) return emulator;
453+
return GetEnv("CLOUD_STORAGE_TESTBENCH_ENDPOINT");
454+
}
455+
456+
std::size_t DefaultConnectionPoolSize() {
457+
std::size_t nthreads = std::thread::hardware_concurrency();
458+
if (nthreads == 0) {
459+
return 4;
460+
}
461+
return 4 * nthreads;
462+
}
463+
464+
} // namespace
465+
390466
namespace internal {
391467

468+
Options ApplyPolicy(Options opts, RetryPolicy const& p) {
469+
opts.set<RetryPolicyOption>(p.clone());
470+
return opts;
471+
}
472+
473+
Options ApplyPolicy(Options opts, BackoffPolicy const& p) {
474+
opts.set<BackoffPolicyOption>(p.clone());
475+
return opts;
476+
}
477+
478+
Options ApplyPolicy(Options opts, IdempotencyPolicy const& p) {
479+
opts.set<IdempotencyPolicyOption>(p.clone());
480+
return opts;
481+
}
482+
483+
Options DefaultOptions(std::shared_ptr<oauth2::Credentials> credentials,
484+
Options opts) {
485+
auto ud = GetEnv("GOOGLE_CLOUD_UNIVERSE_DOMAIN");
486+
if (ud && !ud->empty()) {
487+
opts.set<google::cloud::internal::UniverseDomainOption>(*std::move(ud));
488+
}
489+
auto gcs_ep = google::cloud::internal::UniverseDomainEndpoint(
490+
"https://storage.googleapis.com", opts);
491+
auto iam_ep = absl::StrCat(google::cloud::internal::UniverseDomainEndpoint(
492+
"https://iamcredentials.googleapis.com", opts),
493+
"/v1");
494+
auto o =
495+
Options{}
496+
.set<Oauth2CredentialsOption>(std::move(credentials))
497+
.set<RestEndpointOption>(std::move(gcs_ep))
498+
.set<IamEndpointOption>(std::move(iam_ep))
499+
.set<TargetApiVersionOption>("v1")
500+
.set<ConnectionPoolSizeOption>(DefaultConnectionPoolSize())
501+
.set<DownloadBufferSizeOption>(
502+
GOOGLE_CLOUD_CPP_STORAGE_DEFAULT_DOWNLOAD_BUFFER_SIZE)
503+
.set<UploadBufferSizeOption>(
504+
GOOGLE_CLOUD_CPP_STORAGE_DEFAULT_UPLOAD_BUFFER_SIZE)
505+
.set<MaximumSimpleUploadSizeOption>(
506+
GOOGLE_CLOUD_CPP_STORAGE_DEFAULT_MAXIMUM_SIMPLE_UPLOAD_SIZE)
507+
.set<EnableCurlSslLockingOption>(true)
508+
.set<EnableCurlSigpipeHandlerOption>(true)
509+
.set<MaximumCurlSocketRecvSizeOption>(0)
510+
.set<MaximumCurlSocketSendSizeOption>(0)
511+
.set<TransferStallTimeoutOption>(std::chrono::seconds(
512+
GOOGLE_CLOUD_CPP_STORAGE_DEFAULT_DOWNLOAD_STALL_TIMEOUT))
513+
.set<TransferStallMinimumRateOption>(1)
514+
.set<DownloadStallMinimumRateOption>(1)
515+
.set<RetryPolicyOption>(
516+
LimitedTimeRetryPolicy(
517+
STORAGE_CLIENT_DEFAULT_MAXIMUM_RETRY_PERIOD)
518+
.clone())
519+
.set<BackoffPolicyOption>(
520+
ExponentialBackoffPolicy(
521+
STORAGE_CLIENT_DEFAULT_INITIAL_BACKOFF_DELAY,
522+
STORAGE_CLIENT_DEFAULT_MAXIMUM_BACKOFF_DELAY,
523+
STORAGE_CLIENT_DEFAULT_BACKOFF_SCALING)
524+
.clone())
525+
.set<IdempotencyPolicyOption>(AlwaysRetryIdempotencyPolicy().clone());
526+
527+
o = google::cloud::internal::MergeOptions(std::move(opts), std::move(o));
528+
// If the application did not set `DownloadStallTimeoutOption` then use the
529+
// same value as `TransferStallTimeoutOption` (which could be the default
530+
// value). Some applications need tighter timeouts for downloads, but longer
531+
// timeouts for other transfers.
532+
if (!o.has<DownloadStallTimeoutOption>()) {
533+
o.set<DownloadStallTimeoutOption>(o.get<TransferStallTimeoutOption>());
534+
}
535+
536+
auto emulator = GetEmulator();
537+
if (emulator.has_value()) {
538+
o.set<RestEndpointOption>(*emulator).set<IamEndpointOption>(*emulator +
539+
"/iamapi");
540+
}
541+
542+
auto logging = GetEnv("CLOUD_STORAGE_ENABLE_TRACING");
543+
if (logging) {
544+
for (auto c : absl::StrSplit(*logging, ',')) {
545+
GCP_LOG(INFO) << "Enabling logging for " << c;
546+
o.lookup<LoggingComponentsOption>().insert(std::string(c));
547+
}
548+
}
549+
550+
auto tracing = GetEnv("GOOGLE_CLOUD_CPP_OPENTELEMETRY_TRACING");
551+
if (tracing && !tracing->empty()) {
552+
o.set<OpenTelemetryTracingOption>(true);
553+
}
554+
555+
auto project_id = GetEnv("GOOGLE_CLOUD_PROJECT");
556+
if (project_id.has_value()) {
557+
o.set<ProjectIdOption>(std::move(*project_id));
558+
}
559+
560+
// Always apply the RestClient defaults, even if it is not in use. Now that we
561+
// use the low-level initialization code in
562+
// google/cloud/internal/curl_wrappers.cc, these are always needed.
563+
namespace rest = ::google::cloud::rest_internal;
564+
auto rest_defaults = Options{}
565+
.set<rest::DownloadStallTimeoutOption>(
566+
o.get<DownloadStallTimeoutOption>())
567+
.set<rest::DownloadStallMinimumRateOption>(
568+
o.get<DownloadStallMinimumRateOption>())
569+
.set<rest::TransferStallTimeoutOption>(
570+
o.get<TransferStallTimeoutOption>())
571+
.set<rest::TransferStallMinimumRateOption>(
572+
o.get<TransferStallMinimumRateOption>())
573+
.set<rest::MaximumCurlSocketRecvSizeOption>(
574+
o.get<MaximumCurlSocketRecvSizeOption>())
575+
.set<rest::MaximumCurlSocketSendSizeOption>(
576+
o.get<MaximumCurlSocketSendSizeOption>())
577+
.set<rest::ConnectionPoolSizeOption>(
578+
o.get<ConnectionPoolSizeOption>())
579+
.set<rest::EnableCurlSslLockingOption>(
580+
o.get<EnableCurlSslLockingOption>())
581+
.set<rest::EnableCurlSigpipeHandlerOption>(
582+
o.get<EnableCurlSigpipeHandlerOption>());
583+
584+
// These two are not always present, but if they are, and only if they are, we
585+
// need to map their value to the corresponding option in `rest_internal::`.
586+
if (o.has<storage_experimental::HttpVersionOption>()) {
587+
rest_defaults.set<rest::HttpVersionOption>(
588+
o.get<storage_experimental::HttpVersionOption>());
589+
}
590+
if (o.has<internal::CAPathOption>()) {
591+
rest_defaults.set<rest::CAPathOption>(o.get<internal::CAPathOption>());
592+
}
593+
594+
return google::cloud::internal::MergeOptions(std::move(o),
595+
std::move(rest_defaults));
596+
}
597+
598+
Options DefaultOptionsWithCredentials(Options opts) {
599+
if (opts.has<Oauth2CredentialsOption>()) {
600+
auto credentials = opts.get<Oauth2CredentialsOption>();
601+
return internal::DefaultOptions(std::move(credentials), std::move(opts));
602+
}
603+
if (opts.has<UnifiedCredentialsOption>()) {
604+
auto credentials =
605+
internal::MapCredentials(*opts.get<UnifiedCredentialsOption>());
606+
return internal::DefaultOptions(std::move(credentials), std::move(opts));
607+
}
608+
if (GetEmulator().has_value()) {
609+
return internal::DefaultOptions(
610+
internal::MapCredentials(*google::cloud::MakeInsecureCredentials()),
611+
std::move(opts));
612+
}
613+
auto credentials =
614+
internal::MapCredentials(*google::cloud::MakeGoogleDefaultCredentials(
615+
google::cloud::internal::MakeAuthOptions(opts)));
616+
return internal::DefaultOptions(std::move(credentials), std::move(opts));
617+
}
618+
392619
Client ClientImplDetails::CreateWithDecorations(
393620
Options const& opts, std::shared_ptr<StorageConnection> connection) {
394621
return Client(

‎google/cloud/storage/client.h‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515
#ifndef GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_CLIENT_H
1616
#define GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_CLIENT_H
1717

18-
#include "google/cloud/storage/client_options.h"
1918
#include "google/cloud/storage/hmac_key_metadata.h"
2019
#include "google/cloud/storage/internal/policy_document_request.h"
2120
#include "google/cloud/storage/internal/request_project_id.h"
@@ -55,6 +54,23 @@ namespace internal {
5554
class NonResumableParallelUploadState;
5655
class ResumableParallelUploadState;
5756
struct ClientImplDetails;
57+
58+
Options ApplyPolicy(Options opts, RetryPolicy const& p);
59+
Options ApplyPolicy(Options opts, BackoffPolicy const& p);
60+
Options ApplyPolicy(Options opts, IdempotencyPolicy const& p);
61+
62+
inline Options ApplyPolicies(Options opts) { return opts; }
63+
64+
template <typename P, typename... Policies>
65+
Options ApplyPolicies(Options opts, P&& head, Policies&&... tail) {
66+
opts = ApplyPolicy(std::move(opts), std::forward<P>(head));
67+
return ApplyPolicies(std::move(opts), std::forward<Policies>(tail)...);
68+
}
69+
70+
Options DefaultOptions(std::shared_ptr<oauth2::Credentials> credentials,
71+
Options opts);
72+
Options DefaultOptionsWithCredentials(Options opts);
73+
5874
} // namespace internal
5975
/**
6076
* The Google Cloud Storage (GCS) Client.

0 commit comments

Comments
 (0)