Skip to content

Commit e5c3d98

Browse files
Jinming-HuCopilot
Jinming-Hu
and
Copilot
authored
Storage: introduce StorageRetryPolicy and shared pipeline helper (#7195)
* use common storage pipeline Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Comment why retry-attempt guard is intentional Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent b92f4d9 commit e5c3d98

17 files changed

Lines changed: 531 additions & 468 deletions

File tree

‎sdk/storage/azure-storage-blobs/src/blob_client.cpp‎

Lines changed: 29 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,7 @@
1717
#include <azure/storage/common/internal/reliable_stream.hpp>
1818
#include <azure/storage/common/internal/shared_key_policy.hpp>
1919
#include <azure/storage/common/internal/storage_bearer_token_auth.hpp>
20-
#include <azure/storage/common/internal/storage_per_retry_policy.hpp>
21-
#include <azure/storage/common/internal/storage_service_version_policy.hpp>
20+
#include <azure/storage/common/internal/storage_pipeline.hpp>
2221
#include <azure/storage/common/internal/storage_switch_to_secondary_policy.hpp>
2322
#include <azure/storage/common/internal/structured_message_decoding_stream.hpp>
2423
#include <azure/storage/common/storage_common.hpp>
@@ -59,23 +58,16 @@ namespace Azure { namespace Storage { namespace Blobs {
5958
const BlobClientOptions& options)
6059
: BlobClient(blobUrl, options)
6160
{
62-
BlobClientOptions newOptions = options;
63-
newOptions.PerRetryPolicies.emplace_back(
64-
std::make_unique<_internal::SharedKeyPolicy>(credential));
65-
66-
std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> perRetryPolicies;
67-
std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> perOperationPolicies;
68-
perRetryPolicies.emplace_back(std::make_unique<_internal::StorageSwitchToSecondaryPolicy>(
69-
m_blobUrl.GetHost(), newOptions.SecondaryHostForRetryReads));
70-
perRetryPolicies.emplace_back(std::make_unique<_internal::StoragePerRetryPolicy>());
71-
perOperationPolicies.emplace_back(
72-
std::make_unique<_internal::StorageServiceVersionPolicy>(newOptions.ApiVersion));
61+
_internal::BuildStoragePipelineOptions pipelineOptions;
62+
pipelineOptions.PackageName = _internal::BlobServicePackageName;
63+
pipelineOptions.PackageVersion = _detail::PackageVersion::ToString();
64+
pipelineOptions.PrimaryHost = m_blobUrl.GetHost();
65+
pipelineOptions.SecondaryHost = options.SecondaryHostForRetryReads;
66+
pipelineOptions.ApiVersion = options.ApiVersion;
67+
pipelineOptions.SharedKeyAuthPolicy = std::make_unique<_internal::SharedKeyPolicy>(credential);
68+
7369
m_pipeline = std::make_shared<Azure::Core::Http::_internal::HttpPipeline>(
74-
newOptions,
75-
_internal::BlobServicePackageName,
76-
_detail::PackageVersion::ToString(),
77-
std::move(perRetryPolicies),
78-
std::move(perOperationPolicies));
70+
_internal::BuildHttpPipelinePolicies(options, std::move(pipelineOptions)));
7971
}
8072

8173
BlobClient::BlobClient(
@@ -84,29 +76,25 @@ namespace Azure { namespace Storage { namespace Blobs {
8476
const BlobClientOptions& options)
8577
: BlobClient(blobUrl, options)
8678
{
87-
std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> perRetryPolicies;
88-
std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> perOperationPolicies;
89-
perRetryPolicies.emplace_back(std::make_unique<_internal::StorageSwitchToSecondaryPolicy>(
90-
m_blobUrl.GetHost(), options.SecondaryHostForRetryReads));
91-
perRetryPolicies.emplace_back(std::make_unique<_internal::StoragePerRetryPolicy>());
79+
_internal::BuildStoragePipelineOptions pipelineOptions;
80+
pipelineOptions.PackageName = _internal::BlobServicePackageName;
81+
pipelineOptions.PackageVersion = _detail::PackageVersion::ToString();
82+
pipelineOptions.PrimaryHost = m_blobUrl.GetHost();
83+
pipelineOptions.SecondaryHost = options.SecondaryHostForRetryReads;
84+
pipelineOptions.ApiVersion = options.ApiVersion;
9285
{
9386
Azure::Core::Credentials::TokenRequestContext tokenContext;
9487
tokenContext.Scopes.emplace_back(
9588
options.Audience.HasValue()
9689
? _internal::GetDefaultScopeForAudience(options.Audience.Value().ToString())
9790
: _internal::StorageScope);
98-
perRetryPolicies.emplace_back(
99-
std::make_unique<_internal::StorageBearerTokenAuthenticationPolicy>(
100-
credential, tokenContext, options.EnableTenantDiscovery));
91+
pipelineOptions.TokenAuthPolicy
92+
= std::make_unique<_internal::StorageBearerTokenAuthenticationPolicy>(
93+
credential, tokenContext, options.EnableTenantDiscovery);
10194
}
102-
perOperationPolicies.emplace_back(
103-
std::make_unique<_internal::StorageServiceVersionPolicy>(options.ApiVersion));
95+
10496
m_pipeline = std::make_shared<Azure::Core::Http::_internal::HttpPipeline>(
105-
options,
106-
_internal::BlobServicePackageName,
107-
_detail::PackageVersion::ToString(),
108-
std::move(perRetryPolicies),
109-
std::move(perOperationPolicies));
97+
_internal::BuildHttpPipelinePolicies(options, std::move(pipelineOptions)));
11098
}
11199

112100
BlobClient::BlobClient(const std::string& blobUrl, const BlobClientOptions& options)
@@ -115,19 +103,15 @@ namespace Azure { namespace Storage { namespace Blobs {
115103
m_uploadValidationOptions(options.UploadValidationOptions),
116104
m_downloadValidationOptions(options.DownloadValidationOptions)
117105
{
118-
std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> perRetryPolicies;
119-
std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> perOperationPolicies;
120-
perRetryPolicies.emplace_back(std::make_unique<_internal::StorageSwitchToSecondaryPolicy>(
121-
m_blobUrl.GetHost(), options.SecondaryHostForRetryReads));
122-
perRetryPolicies.emplace_back(std::make_unique<_internal::StoragePerRetryPolicy>());
123-
perOperationPolicies.emplace_back(
124-
std::make_unique<_internal::StorageServiceVersionPolicy>(options.ApiVersion));
106+
_internal::BuildStoragePipelineOptions pipelineOptions;
107+
pipelineOptions.PackageName = _internal::BlobServicePackageName;
108+
pipelineOptions.PackageVersion = _detail::PackageVersion::ToString();
109+
pipelineOptions.PrimaryHost = m_blobUrl.GetHost();
110+
pipelineOptions.SecondaryHost = options.SecondaryHostForRetryReads;
111+
pipelineOptions.ApiVersion = options.ApiVersion;
112+
125113
m_pipeline = std::make_shared<Azure::Core::Http::_internal::HttpPipeline>(
126-
options,
127-
_internal::BlobServicePackageName,
128-
_detail::PackageVersion::ToString(),
129-
std::move(perRetryPolicies),
130-
std::move(perOperationPolicies));
114+
_internal::BuildHttpPipelinePolicies(options, std::move(pipelineOptions)));
131115
}
132116

133117
BlockBlobClient BlobClient::AsBlockBlobClient() const { return BlockBlobClient(*this); }

‎sdk/storage/azure-storage-blobs/src/blob_container_client.cpp‎

Lines changed: 29 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
#include <azure/storage/common/internal/shared_key_policy.hpp>
1616
#include <azure/storage/common/internal/storage_bearer_token_auth.hpp>
1717
#include <azure/storage/common/internal/storage_per_retry_policy.hpp>
18+
#include <azure/storage/common/internal/storage_pipeline.hpp>
1819
#include <azure/storage/common/internal/storage_service_version_policy.hpp>
1920
#include <azure/storage/common/internal/storage_switch_to_secondary_policy.hpp>
2021
#include <azure/storage/common/storage_common.hpp>
@@ -145,14 +146,18 @@ namespace Azure { namespace Storage { namespace Blobs {
145146
= _detail::ConstructBatchRequestPolicy(perRetryPolicies, perOperationPolicies, newOptions);
146147

147148
m_batchSubrequestPipeline
148-
= _detail::ConstructBatchSubrequestPolicy(nullptr, std::move(sharedKeyAuthPolicy), options);
149+
= _detail::ConstructBatchSubrequestPolicy(nullptr, sharedKeyAuthPolicy->Clone(), options);
150+
151+
_internal::BuildStoragePipelineOptions pipelineOptions;
152+
pipelineOptions.PackageName = _internal::BlobServicePackageName;
153+
pipelineOptions.PackageVersion = _detail::PackageVersion::ToString();
154+
pipelineOptions.PrimaryHost = m_blobContainerUrl.GetHost();
155+
pipelineOptions.SecondaryHost = options.SecondaryHostForRetryReads;
156+
pipelineOptions.ApiVersion = options.ApiVersion;
157+
pipelineOptions.SharedKeyAuthPolicy = std::move(sharedKeyAuthPolicy);
149158

150159
m_pipeline = std::make_shared<Azure::Core::Http::_internal::HttpPipeline>(
151-
newOptions,
152-
_internal::BlobServicePackageName,
153-
_detail::PackageVersion::ToString(),
154-
std::move(perRetryPolicies),
155-
std::move(perOperationPolicies));
160+
_internal::BuildHttpPipelinePolicies(options, std::move(pipelineOptions)));
156161
}
157162

158163
BlobContainerClient::BlobContainerClient(
@@ -184,14 +189,18 @@ namespace Azure { namespace Storage { namespace Blobs {
184189
= _detail::ConstructBatchRequestPolicy(perRetryPolicies, perOperationPolicies, options);
185190

186191
m_batchSubrequestPipeline
187-
= _detail::ConstructBatchSubrequestPolicy(std::move(tokenAuthPolicy), nullptr, options);
192+
= _detail::ConstructBatchSubrequestPolicy(tokenAuthPolicy->Clone(), nullptr, options);
193+
194+
_internal::BuildStoragePipelineOptions pipelineOptions;
195+
pipelineOptions.PackageName = _internal::BlobServicePackageName;
196+
pipelineOptions.PackageVersion = _detail::PackageVersion::ToString();
197+
pipelineOptions.PrimaryHost = m_blobContainerUrl.GetHost();
198+
pipelineOptions.SecondaryHost = options.SecondaryHostForRetryReads;
199+
pipelineOptions.ApiVersion = options.ApiVersion;
200+
pipelineOptions.TokenAuthPolicy = std::move(tokenAuthPolicy);
188201

189202
m_pipeline = std::make_shared<Azure::Core::Http::_internal::HttpPipeline>(
190-
options,
191-
_internal::BlobServicePackageName,
192-
_detail::PackageVersion::ToString(),
193-
std::move(perRetryPolicies),
194-
std::move(perOperationPolicies));
203+
_internal::BuildHttpPipelinePolicies(options, std::move(pipelineOptions)));
195204
}
196205

197206
BlobContainerClient::BlobContainerClient(
@@ -215,12 +224,15 @@ namespace Azure { namespace Storage { namespace Blobs {
215224

216225
m_batchSubrequestPipeline = _detail::ConstructBatchSubrequestPolicy(nullptr, nullptr, options);
217226

227+
_internal::BuildStoragePipelineOptions pipelineOptions;
228+
pipelineOptions.PackageName = _internal::BlobServicePackageName;
229+
pipelineOptions.PackageVersion = _detail::PackageVersion::ToString();
230+
pipelineOptions.PrimaryHost = m_blobContainerUrl.GetHost();
231+
pipelineOptions.SecondaryHost = options.SecondaryHostForRetryReads;
232+
pipelineOptions.ApiVersion = options.ApiVersion;
233+
218234
m_pipeline = std::make_shared<Azure::Core::Http::_internal::HttpPipeline>(
219-
options,
220-
_internal::BlobServicePackageName,
221-
_detail::PackageVersion::ToString(),
222-
std::move(perRetryPolicies),
223-
std::move(perOperationPolicies));
235+
_internal::BuildHttpPipelinePolicies(options, std::move(pipelineOptions)));
224236
}
225237

226238
BlobClient BlobContainerClient::GetBlobClient(const std::string& blobName) const

‎sdk/storage/azure-storage-blobs/src/blob_service_client.cpp‎

Lines changed: 29 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
#include <azure/storage/common/internal/shared_key_policy.hpp>
1313
#include <azure/storage/common/internal/storage_bearer_token_auth.hpp>
1414
#include <azure/storage/common/internal/storage_per_retry_policy.hpp>
15+
#include <azure/storage/common/internal/storage_pipeline.hpp>
1516
#include <azure/storage/common/internal/storage_service_version_policy.hpp>
1617
#include <azure/storage/common/internal/storage_switch_to_secondary_policy.hpp>
1718
#include <azure/storage/common/storage_common.hpp>
@@ -58,14 +59,18 @@ namespace Azure { namespace Storage { namespace Blobs {
5859
= _detail::ConstructBatchRequestPolicy(perRetryPolicies, perOperationPolicies, newOptions);
5960

6061
m_batchSubrequestPipeline
61-
= _detail::ConstructBatchSubrequestPolicy(nullptr, std::move(sharedKeyPolicy), options);
62+
= _detail::ConstructBatchSubrequestPolicy(nullptr, sharedKeyPolicy->Clone(), options);
63+
64+
_internal::BuildStoragePipelineOptions pipelineOptions;
65+
pipelineOptions.PackageName = _internal::BlobServicePackageName;
66+
pipelineOptions.PackageVersion = _detail::PackageVersion::ToString();
67+
pipelineOptions.PrimaryHost = m_serviceUrl.GetHost();
68+
pipelineOptions.SecondaryHost = options.SecondaryHostForRetryReads;
69+
pipelineOptions.ApiVersion = options.ApiVersion;
70+
pipelineOptions.SharedKeyAuthPolicy = std::move(sharedKeyPolicy);
6271

6372
m_pipeline = std::make_shared<Azure::Core::Http::_internal::HttpPipeline>(
64-
newOptions,
65-
_internal::BlobServicePackageName,
66-
_detail::PackageVersion::ToString(),
67-
std::move(perRetryPolicies),
68-
std::move(perOperationPolicies));
73+
_internal::BuildHttpPipelinePolicies(options, std::move(pipelineOptions)));
6974
}
7075

7176
BlobServiceClient::BlobServiceClient(
@@ -97,14 +102,18 @@ namespace Azure { namespace Storage { namespace Blobs {
97102
= _detail::ConstructBatchRequestPolicy(perRetryPolicies, perOperationPolicies, options);
98103

99104
m_batchSubrequestPipeline
100-
= _detail::ConstructBatchSubrequestPolicy(std::move(tokenAuthPolicy), nullptr, options);
105+
= _detail::ConstructBatchSubrequestPolicy(tokenAuthPolicy->Clone(), nullptr, options);
106+
107+
_internal::BuildStoragePipelineOptions pipelineOptions;
108+
pipelineOptions.PackageName = _internal::BlobServicePackageName;
109+
pipelineOptions.PackageVersion = _detail::PackageVersion::ToString();
110+
pipelineOptions.PrimaryHost = m_serviceUrl.GetHost();
111+
pipelineOptions.SecondaryHost = options.SecondaryHostForRetryReads;
112+
pipelineOptions.ApiVersion = options.ApiVersion;
113+
pipelineOptions.TokenAuthPolicy = std::move(tokenAuthPolicy);
101114

102115
m_pipeline = std::make_shared<Azure::Core::Http::_internal::HttpPipeline>(
103-
options,
104-
_internal::BlobServicePackageName,
105-
_detail::PackageVersion::ToString(),
106-
std::move(perRetryPolicies),
107-
std::move(perOperationPolicies));
116+
_internal::BuildHttpPipelinePolicies(options, std::move(pipelineOptions)));
108117
}
109118

110119
BlobServiceClient::BlobServiceClient(
@@ -128,12 +137,15 @@ namespace Azure { namespace Storage { namespace Blobs {
128137

129138
m_batchSubrequestPipeline = _detail::ConstructBatchSubrequestPolicy(nullptr, nullptr, options);
130139

140+
_internal::BuildStoragePipelineOptions pipelineOptions;
141+
pipelineOptions.PackageName = _internal::BlobServicePackageName;
142+
pipelineOptions.PackageVersion = _detail::PackageVersion::ToString();
143+
pipelineOptions.PrimaryHost = m_serviceUrl.GetHost();
144+
pipelineOptions.SecondaryHost = options.SecondaryHostForRetryReads;
145+
pipelineOptions.ApiVersion = options.ApiVersion;
146+
131147
m_pipeline = std::make_shared<Azure::Core::Http::_internal::HttpPipeline>(
132-
options,
133-
_internal::BlobServicePackageName,
134-
_detail::PackageVersion::ToString(),
135-
std::move(perRetryPolicies),
136-
std::move(perOperationPolicies));
148+
_internal::BuildHttpPipelinePolicies(options, std::move(pipelineOptions)));
137149
}
138150

139151
BlobContainerClient BlobServiceClient::GetBlobContainerClient(

‎sdk/storage/azure-storage-common/CMakeLists.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,8 @@ set(
5555
inc/azure/storage/common/internal/storage_bearer_token_auth.hpp
5656
inc/azure/storage/common/internal/storage_bearer_token_authentication_policy.hpp
5757
inc/azure/storage/common/internal/storage_per_retry_policy.hpp
58+
inc/azure/storage/common/internal/storage_pipeline.hpp
59+
inc/azure/storage/common/internal/storage_retry_policy.hpp
5860
inc/azure/storage/common/internal/storage_service_version_policy.hpp
5961
inc/azure/storage/common/internal/storage_switch_to_secondary_policy.hpp
6062
inc/azure/storage/common/internal/structured_message_decoding_stream.hpp
@@ -80,6 +82,8 @@ set(
8082
src/storage_credential.cpp
8183
src/storage_exception.cpp
8284
src/storage_per_retry_policy.cpp
85+
src/storage_pipeline.cpp
86+
src/storage_retry_policy.cpp
8387
src/storage_switch_to_secondary_policy.cpp
8488
src/structured_message_decoding_stream.cpp
8589
src/structured_message_encoding_stream.cpp

‎sdk/storage/azure-storage-common/inc/azure/storage/common/internal/storage_pipeline.hpp‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
#pragma once
5+
6+
#include <azure/core/http/policies/policy.hpp>
7+
#include <azure/core/internal/client_options.hpp>
8+
9+
#include <memory>
10+
#include <string>
11+
#include <vector>
12+
13+
namespace Azure { namespace Storage { namespace _internal {
14+
15+
struct BuildStoragePipelineOptions final
16+
{
17+
std::string PackageName;
18+
std::string PackageVersion;
19+
std::string PrimaryHost;
20+
std::string SecondaryHost;
21+
std::string ApiVersion;
22+
std::unique_ptr<Core::Http::Policies::HttpPolicy> TokenAuthPolicy;
23+
std::unique_ptr<Core::Http::Policies::HttpPolicy> SharedKeyAuthPolicy;
24+
};
25+
26+
std::vector<std::unique_ptr<Core::Http::Policies::HttpPolicy>> BuildHttpPipelinePolicies(
27+
const Core::_internal::ClientOptions& clientOptions,
28+
BuildStoragePipelineOptions storagePipelineOptions);
29+
30+
}}} // namespace Azure::Storage::_internal

‎sdk/storage/azure-storage-common/inc/azure/storage/common/internal/storage_retry_policy.hpp‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
#pragma once
5+
6+
#include <azure/core/http/policies/policy.hpp>
7+
8+
#include <chrono>
9+
#include <cstdint>
10+
#include <memory>
11+
12+
namespace Azure { namespace Storage { namespace _internal {
13+
14+
class StorageRetryPolicy final : public Core::Http::Policies::_internal::RetryPolicyBase {
15+
public:
16+
explicit StorageRetryPolicy(Core::Http::Policies::RetryOptions options)
17+
: Core::Http::Policies::_internal::RetryPolicyBase(std::move(options))
18+
{
19+
}
20+
21+
std::unique_ptr<Core::Http::Policies::HttpPolicy> Clone() const override
22+
{
23+
return std::make_unique<StorageRetryPolicy>(*this);
24+
}
25+
26+
protected:
27+
bool ShouldRetryOnResponse(
28+
Core::Http::RawResponse const& response,
29+
Core::Http::Policies::RetryOptions const& retryOptions,
30+
int32_t attempt,
31+
std::chrono::milliseconds& retryAfter,
32+
double jitterFactor = -1) const override;
33+
};
34+
35+
}}} // namespace Azure::Storage::_internal

0 commit comments

Comments
 (0)