Skip to content

Commit 51de7e8

Browse files
photonlibosMoonpuck
photonlibos
and
Moonpuck
authored
Return ETag in PutObject, AppendObject and CompleteMultipart (alibaba#1123) (alibaba#1124)
* Return ETag in PutObject, AppendObject and CompleteMultipart Co-authored-by: Hongren Lin <lindong941107@gmail.com>
1 parent 2d12c4e commit 51de7e8

3 files changed

Lines changed: 121 additions & 18 deletions

File tree

‎ecosystem/oss.cpp‎

Lines changed: 30 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -287,7 +287,7 @@ static int do_http_call(HTTP_STACK_OP& op, const ClientOptions& options,
287287
}
288288

289289
static int verify_crc64_if_needed(HTTP_STACK_OP& op, std::string_view object,
290-
uint64_t* expected_crc64) {
290+
const uint64_t* expected_crc64) {
291291
if (!expected_crc64) return 0;
292292
auto it = op.resp.headers.find("x-oss-hash-crc64ecma");
293293
if (it == op.resp.headers.end()) {
@@ -337,6 +337,7 @@ class OssClientImpl : public Client {
337337

338338
int fill_meta(HTTP_STACK_OP& op, ObjectMeta& meta);
339339
int fill_meta(HTTP_STACK_OP& op, ObjectHeaderMeta& meta);
340+
int fill_upload_response(HTTP_STACK_OP& op, ObjectUploadOptions& opts);
340341

341342
ssize_t get_object_range(std::string_view object, const struct iovec* iov,
342343
int iovcnt, off_t offset,
@@ -345,24 +346,24 @@ class OssClientImpl : public Client {
345346
int batch_get_objects(std::vector<GetObjectParameters>& params);
346347

347348
ssize_t put_object(std::string_view object, const struct iovec* iov,
348-
int iovcnt, uint64_t* expected_crc64 = nullptr);
349+
int iovcnt, ObjectUploadOptions& opts);
349350

350351
ssize_t append_object(std::string_view object, const struct iovec* iov,
351352
int iovcnt, off_t position,
352-
uint64_t* expected_crc64 = nullptr);
353+
ObjectUploadOptions& opts);
353354

354355
int copy_object(std::string_view src_object, std::string_view dst_object,
355356
bool overwrite = false, bool set_mime = false);
356357

357358
int init_multipart_upload(std::string_view object, void** context);
358359

359360
ssize_t upload_part(void* context, const struct iovec* iov, int iovcnt,
360-
int part_number, uint64_t* expected_crc64 = nullptr);
361+
int part_number, ObjectUploadOptions& opts);
361362

362363
int upload_part_copy(void* context, off_t offset, size_t count,
363364
int part_number, std::string_view from = {});
364365

365-
int complete_multipart_upload(void* context, uint64_t* expected_crc64);
366+
int complete_multipart_upload(void* context, ObjectUploadOptions& opts);
366367

367368
int abort_multipart_upload(void* context);
368369

@@ -849,6 +850,17 @@ int OssClient::fill_meta(HTTP_STACK_OP& op, ObjectMeta& meta) {
849850
return 0;
850851
}
851852

853+
int OssClient::fill_upload_response(HTTP_STACK_OP& op,
854+
ObjectUploadOptions& opts) {
855+
if (opts.etag) {
856+
auto it = op.resp.headers.find("ETag");
857+
if (it != op.resp.headers.end()) {
858+
opts.etag->assign(it.second().data(), it.second().size());
859+
}
860+
}
861+
return 0;
862+
}
863+
852864
ssize_t OssClient::get_object_range(std::string_view obj_path,
853865
const struct iovec* iov, int iovcnt,
854866
off_t offset, ObjectHeaderMeta* meta) {
@@ -1200,7 +1212,7 @@ int OssClient::batch_get_objects(std::vector<GetObjectParameters>& params) {
12001212
}
12011213

12021214
ssize_t OssClient::put_object(std::string_view object, const struct iovec* iov,
1203-
int iovcnt, uint64_t* expected_crc64) {
1215+
int iovcnt, ObjectUploadOptions& opts) {
12041216
iovector_view view((struct iovec*)iov, iovcnt);
12051217
auto cnt = view.sum();
12061218

@@ -1215,14 +1227,15 @@ ssize_t OssClient::put_object(std::string_view object, const struct iovec* iov,
12151227
op.body_writer = {&view, &body_writer_cb};
12161228
int r = sign_and_call(op, Verb::PUT, oss_url);
12171229
if (r < 0) return r;
1218-
r = verify_crc64_if_needed(op, oss_url.object(), expected_crc64);
1230+
r = verify_crc64_if_needed(op, oss_url.object(), opts.expected_crc64);
12191231
if (r < 0) return r;
1232+
fill_upload_response(op, opts);
12201233
return cnt;
12211234
}
12221235

12231236
ssize_t OssClient::append_object(std::string_view object,
12241237
const struct iovec* iov, int iovcnt,
1225-
off_t position, uint64_t* expected_crc64) {
1238+
off_t position, ObjectUploadOptions& opts) {
12261239
iovector_view view((struct iovec*)iov, iovcnt);
12271240
auto cnt = view.sum();
12281241

@@ -1244,8 +1257,9 @@ ssize_t OssClient::append_object(std::string_view object,
12441257
op.body_writer = {&view, &body_writer_cb};
12451258
int r = sign_and_call(op, Verb::POST, oss_url, query_params);
12461259
if (r < 0) return r;
1247-
r = verify_crc64_if_needed(op, oss_url.object(), expected_crc64);
1260+
r = verify_crc64_if_needed(op, oss_url.object(), opts.expected_crc64);
12481261
if (r < 0) return r;
1262+
fill_upload_response(op, opts);
12491263
return cnt;
12501264
}
12511265

@@ -1297,7 +1311,7 @@ int OssClient::init_multipart_upload(std::string_view object, void** context) {
12971311

12981312
ssize_t OssClient::upload_part(void* context, const struct iovec* iov,
12991313
int iovcnt, int part_number,
1300-
uint64_t* expected_crc64) {
1314+
ObjectUploadOptions& opts) {
13011315
iovector_view view((struct iovec*)iov, iovcnt);
13021316
auto cnt = view.sum();
13031317
assert(cnt > 0);
@@ -1327,7 +1341,7 @@ ssize_t OssClient::upload_part(void* context, const struct iovec* iov,
13271341
if (etag.empty())
13281342
LOG_ERROR_RETURN(EINVAL, -1, "unexpected response with empty etag");
13291343

1330-
r = verify_crc64_if_needed(op, oss_url.object(), expected_crc64);
1344+
r = verify_crc64_if_needed(op, oss_url.object(), opts.expected_crc64);
13311345
if (r < 0) return r;
13321346

13331347
SCOPED_LOCK(ctx->lock);
@@ -1381,7 +1395,7 @@ int OssClient::upload_part_copy(void* context, off_t offset, size_t count,
13811395
}
13821396

13831397
int OssClient::complete_multipart_upload(void* context,
1384-
uint64_t* expected_crc64) {
1398+
ObjectUploadOptions& opts) {
13851399
assert(context);
13861400

13871401
oss_multipart_context* ctx = (oss_multipart_context*)context;
@@ -1420,7 +1434,10 @@ int OssClient::complete_multipart_upload(void* context,
14201434
op.body_writer = {&view, &body_writer_cb};
14211435
int r = sign_and_call(op, Verb::POST, oss_url, query_params);
14221436
if (r < 0) return r;
1423-
return verify_crc64_if_needed(op, oss_url.object(), expected_crc64);
1437+
r = verify_crc64_if_needed(op, oss_url.object(), opts.expected_crc64);
1438+
if (r < 0) return r;
1439+
fill_upload_response(op, opts);
1440+
return r;
14241441
}
14251442

14261443
int OssClient::abort_multipart_upload(void* context) {

‎ecosystem/oss.h‎

Lines changed: 33 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,14 @@ struct GetObjectParameters {
140140
int result = -1;
141141
};
142142

143+
struct ObjectUploadOptions {
144+
// inputs
145+
const uint64_t *expected_crc64 = nullptr;
146+
147+
// outputs
148+
std::string *etag = nullptr;
149+
};
150+
143151
class Client : public Object {
144152
public:
145153
virtual int list_objects(std::string_view prefix, ListObjectsCallback cb,
@@ -166,18 +174,28 @@ class Client : public Object {
166174
// return -1.
167175
// if expected_crc64 is specified, we will compare the value with the
168176
// returned object crc64 to validate the object integrity.
177+
ssize_t put_object(std::string_view object, const struct iovec* iov,
178+
int iovcnt, uint64_t* expected_crc64 = nullptr) {
179+
ObjectUploadOptions opts{.expected_crc64 = expected_crc64};
180+
return put_object(object, iov, iovcnt, opts);
181+
};
169182
virtual ssize_t put_object(std::string_view object, const struct iovec* iov,
170-
int iovcnt,
171-
uint64_t* expected_crc64 = nullptr) = 0;
183+
int iovcnt, ObjectUploadOptions& opts) = 0;
172184

173185
// return value is the newly appended size if the operation succeeds,
174186
// otherwise return -1.
175187
// if expected_crc64 is specified, we will compare the value with the
176188
// returned object crc64 to validate the object integrity.
189+
ssize_t append_object(std::string_view object, const struct iovec* iov,
190+
int iovcnt, off_t position,
191+
uint64_t* expected_crc64 = nullptr) {
192+
ObjectUploadOptions opts{.expected_crc64 = expected_crc64};
193+
return append_object(object, iov, iovcnt, position, opts);
194+
};
177195
virtual ssize_t append_object(std::string_view object,
178196
const struct iovec* iov, int iovcnt,
179197
off_t position,
180-
uint64_t* expected_crc64 = nullptr) = 0;
198+
ObjectUploadOptions& opts) = 0;
181199

182200
virtual int copy_object(std::string_view src_object,
183201
std::string_view dst_object, bool overwrite = false,
@@ -190,17 +208,27 @@ class Client : public Object {
190208
// return -1.
191209
// if expected_crc64 is specified, we will compare the value with the
192210
// returned part crc64 to validate the part integrity.
211+
ssize_t upload_part(void* context, const struct iovec* iov, int iovcnt,
212+
int part_number, uint64_t* expected_crc64 = nullptr) {
213+
ObjectUploadOptions opts{.expected_crc64 = expected_crc64};
214+
return upload_part(context, iov, iovcnt, part_number, opts);
215+
};
193216
virtual ssize_t upload_part(void* context, const struct iovec* iov,
194217
int iovcnt, int part_number,
195-
uint64_t* expected_crc64 = nullptr) = 0;
218+
ObjectUploadOptions& opts) = 0;
196219

197220
virtual int upload_part_copy(void* context, off_t offset, size_t count,
198221
int part_number, std::string_view from = {}) = 0;
199222

200223
// if expected_crc64 is specified, we will compare the value with the
201224
// returned object crc64 to validate the object integrity.
225+
int complete_multipart_upload(void* context,
226+
uint64_t* expected_crc64 = nullptr) {
227+
ObjectUploadOptions opts{.expected_crc64 = expected_crc64};
228+
return complete_multipart_upload(context, opts);
229+
}
202230
virtual int complete_multipart_upload(void* context,
203-
uint64_t* expected_crc64) = 0;
231+
ObjectUploadOptions& opts) = 0;
204232

205233
virtual int abort_multipart_upload(void* context) = 0;
206234

‎ecosystem/test/test_oss.cpp‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ class BasicAuthOssTest : public ::testing::Test {
9292
void multipart();
9393
void symlink();
9494
void batch_get_objects();
95+
void upload_with_options();
9596

9697
private:
9798
std::string bucket_prefix_;
@@ -534,6 +535,59 @@ void BasicAuthOssTest::batch_get_objects() {
534535
}
535536
}
536537

538+
void BasicAuthOssTest::upload_with_options() {
539+
// put_object
540+
auto path = get_real_test_path("upload_with_options/testfile");
541+
const size_t file_size = 1025;
542+
char buf[file_size] = {3};
543+
auto crc64 = crc64ecma(buf, file_size, 0);
544+
iovec iov{buf, file_size};
545+
546+
std::string etag;
547+
ObjectUploadOptions opts{&crc64, &etag};
548+
int ret = client->put_object(path, &iov, 1, opts);
549+
ASSERT_EQ(ret, file_size) << "Failed to put object for upload test";
550+
551+
uint64_t invalid_crc64 = 0xdeadbeef;
552+
ObjectUploadOptions opts_wrong_crc64{&invalid_crc64, &etag};
553+
ret = client->put_object(path, &iov, 1, opts_wrong_crc64);
554+
ASSERT_EQ(ret, -1) << "crc64 check did not work";
555+
556+
ObjectMeta meta;
557+
ret = client->get_object_meta(path, meta);
558+
ASSERT_EQ(ret, 0);
559+
EXPECT_TRUE(meta.has_etag());
560+
EXPECT_EQ(meta.etag, etag);
561+
562+
// append_object
563+
path = get_real_test_path("upload_with_options/testfile.append");
564+
565+
ret = client->append_object(path, &iov, 1, 0, opts);
566+
ASSERT_EQ(ret, file_size) << "Failed to append object for upload test";
567+
568+
ret = client->get_object_meta(path, meta);
569+
ASSERT_EQ(ret, 0);
570+
EXPECT_TRUE(meta.has_etag());
571+
EXPECT_EQ(meta.etag, etag);
572+
573+
// multipart
574+
path = get_real_test_path("upload_with_options/testfile.multipart");
575+
void* context = nullptr;
576+
ret = client->init_multipart_upload(path, &context);
577+
ASSERT_EQ(ret, 0);
578+
579+
ret = client->upload_part(context, &iov, 1, 1, opts);
580+
ASSERT_EQ(ret, file_size) << "Failed to upload object for upload test";
581+
582+
ret = client->complete_multipart_upload(context, opts);
583+
ASSERT_EQ(ret, 0);
584+
585+
ret = client->get_object_meta(path, meta);
586+
ASSERT_EQ(ret, 0);
587+
EXPECT_TRUE(meta.has_etag());
588+
EXPECT_EQ(meta.etag, etag);
589+
}
590+
537591
void CachedAuthOssTest::repeatedly_get() {
538592
std::vector<std::string> paths;
539593
const size_t file_size = 1025;
@@ -573,6 +627,8 @@ TEST_F(BasicAuthOssTest, put_and_get_meta) { put_and_get_meta(); }
573627
TEST_F(BasicAuthOssTest, copy_and_rename) { copy_and_rename(); }
574628
TEST_F(BasicAuthOssTest, symlink) { symlink(); }
575629
TEST_F(BasicAuthOssTest, batch_get_objects) { batch_get_objects(); }
630+
631+
TEST_F(BasicAuthOssTest, upload_with_options) { upload_with_options(); }
576632
TEST_F(CachedAuthOssTest, listobjects) { list_objects(); }
577633
TEST_F(CachedAuthOssTest, multipart) { multipart(); }
578634
TEST_F(CachedAuthOssTest, append_and_get) { append_and_get(); }
@@ -581,6 +637,8 @@ TEST_F(CachedAuthOssTest, copy_and_rename) { copy_and_rename(); }
581637
TEST_F(CachedAuthOssTest, repeatedly_get) { repeatedly_get(); }
582638
TEST_F(CachedAuthOssTest, symlink) { symlink(); }
583639
TEST_F(CachedAuthOssTest, batch_get_objects) { batch_get_objects(); }
640+
641+
TEST_F(CachedAuthOssTest, upload_with_options) { upload_with_options(); }
584642
TEST_F(CustomCachedAuthOssTest, repeatedly_get) { repeatedly_get(); }
585643

586644
int main(int argc, char* argv[]) {

0 commit comments

Comments
 (0)