Skip to content

Commit bf92ae2

Browse files
bert-ebert-e
committed
Merge branch 'bugfix/CLDSRV-835-retry-tcp-flakiness' into q/9.3
2 parents 603e27a + c4c44f0 commit bf92ae2

File tree

1 file changed

+36
-26
lines changed

1 file changed

+36
-26
lines changed

‎tests/functional/aws-node-sdk/test/object/copyPart.js‎

Lines changed: 36 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -307,34 +307,44 @@ describe('Object Part Copy', () => {
307307
})).then(initiateRes => {
308308
sourceMpuId = initiateRes.UploadId;
309309
}).catch(err => {
310-
process.stdout.write(`Error initiating MPU ' +
311-
'in MPU beforeEach: ${err}\n`);
310+
process.stdout.write(`Error initiating MPU in MPU beforeEach: ${err}\n`);
312311
throw err;
313312
}).then(() => {
314313
const partUploads = [];
314+
// Concurrent uploads help trigger flakiness with "TimeoutError:
315+
// socket hang up" due to a keep-alive race: the server closes
316+
// an idle connection just as the client picks it from the pool.
317+
const uploadWithRetry = (params, attempt = 0) =>
318+
s3.send(new UploadPartCommand(params)).catch(err => {
319+
if (attempt < 3) {
320+
process.stdout.write(`Retrying UploadPart ${params.PartNumber} `
321+
+ `(attempt ${attempt + 1}/3): ${err}\n`);
322+
return uploadWithRetry(params, attempt + 1);
323+
}
324+
throw err;
325+
});
315326
for (let i = 1; i < 10; i++) {
316327
const partBuffHere = i % 2 ? partBuff : otherPartBuff;
317328
const partHashHere = i % 2 ? partHash : otherPartHash;
318-
partUploads.push(s3.send(new UploadPartCommand({
329+
partUploads.push(uploadWithRetry({
319330
Bucket: sourceBucketName,
320331
Key: sourceMpuKey,
321332
PartNumber: i,
322333
UploadId: sourceMpuId,
323334
Body: partBuffHere,
324-
})));
335+
}));
325336
parts.push({
326337
ETag: partHashHere,
327338
PartNumber: i,
328339
});
329340
}
330-
process.stdout.write('about to put parts');
341+
process.stdout.write('about to put parts\n');
331342
return Promise.all(partUploads);
332343
}).catch(err => {
333-
process.stdout.write(`Error putting parts in ' +
334-
'MPU beforeEach: ${err}\n`);
344+
process.stdout.write(`Error putting parts in MPU beforeEach: ${err}\n`);
335345
throw err;
336346
}).then(() => {
337-
process.stdout.write('completing mpu');
347+
process.stdout.write('completing mpu\n');
338348
return s3.send(new CompleteMultipartUploadCommand({
339349
Bucket: sourceBucketName,
340350
Key: sourceMpuKey,
@@ -344,7 +354,7 @@ describe('Object Part Copy', () => {
344354
},
345355
}));
346356
}).then(() => {
347-
process.stdout.write('finished completing mpu');
357+
process.stdout.write('finished completing mpu\n');
348358
}).catch(err => {
349359
process.stdout.write(`Error in MPU beforeEach: ${err}\n`);
350360
throw err;
@@ -365,7 +375,7 @@ describe('Object Part Copy', () => {
365375

366376
it('should copy a part from a source bucket to a different ' +
367377
'destination bucket', () => {
368-
process.stdout.write('Entered first mpu test');
378+
process.stdout.write('Entered first mpu test\n');
369379
return s3.send(new UploadPartCopyCommand({
370380
Bucket: destBucketName,
371381
Key: destObjName,
@@ -381,7 +391,7 @@ describe('Object Part Copy', () => {
381391

382392
it('should copy two parts from a source bucket to a different ' +
383393
'destination bucket and complete the MPU', () => {
384-
process.stdout.write('Putting first part in MPU test');
394+
process.stdout.write('Putting first part in MPU test\n');
385395
return s3.send(new UploadPartCopyCommand({
386396
Bucket: destBucketName,
387397
Key: destObjName,
@@ -392,7 +402,7 @@ describe('Object Part Copy', () => {
392402
assert.strictEqual(res.CopyPartResult.ETag, totalMpuObjectHash);
393403
assert(res.CopyPartResult.LastModified);
394404
}).then(() => {
395-
process.stdout.write('Putting second part in MPU test');
405+
process.stdout.write('Putting second part in MPU test\n');
396406
return s3.send(new UploadPartCopyCommand({
397407
Bucket: destBucketName,
398408
Key: destObjName,
@@ -403,7 +413,7 @@ describe('Object Part Copy', () => {
403413
assert.strictEqual(res.CopyPartResult.ETag, totalMpuObjectHash);
404414
assert(res.CopyPartResult.LastModified);
405415
}).then(() => {
406-
process.stdout.write('Completing MPU');
416+
process.stdout.write('Completing MPU\n');
407417
return s3.send(new CompleteMultipartUploadCommand({
408418
Bucket: destBucketName,
409419
Key: destObjName,
@@ -431,7 +441,7 @@ describe('Object Part Copy', () => {
431441
it('should copy two parts with range headers from a source ' +
432442
'bucket to a different destination bucket and ' +
433443
'complete the MPU', () => {
434-
process.stdout.write('Putting first part in MPU range test');
444+
process.stdout.write('Putting first part in MPU range test\n');
435445
const part1ETag = '"b1e0d096c8f0670c5367d131e392b84a"';
436446
const part2ETag = '"a2468d5c0ec2d4d5fc13b73beb63080a"';
437447
// combined ETag returned by AWS (combination of part ETags
@@ -449,7 +459,7 @@ describe('Object Part Copy', () => {
449459
assert.strictEqual(res.CopyPartResult.ETag, part1ETag);
450460
assert(res.CopyPartResult.LastModified);
451461
}).then(() => {
452-
process.stdout.write('Putting second part in MPU test');
462+
process.stdout.write('Putting second part in MPU test\n');
453463
return s3.send(new UploadPartCopyCommand({
454464
Bucket: destBucketName,
455465
Key: destObjName,
@@ -461,7 +471,7 @@ describe('Object Part Copy', () => {
461471
assert.strictEqual(res.CopyPartResult.ETag, part2ETag);
462472
assert(res.CopyPartResult.LastModified);
463473
}).then(() => {
464-
process.stdout.write('Completing MPU');
474+
process.stdout.write('Completing MPU\n');
465475
return s3.send(new CompleteMultipartUploadCommand({
466476
Bucket: destBucketName,
467477
Key: destObjName,
@@ -477,7 +487,7 @@ describe('Object Part Copy', () => {
477487
assert.strictEqual(res.Key, destObjName);
478488
assert.strictEqual(res.ETag, finalCombinedETag);
479489
}).then(() => {
480-
process.stdout.write('Getting new object');
490+
process.stdout.write('Getting new object\n');
481491
return s3.send(new GetObjectCommand({
482492
Bucket: destBucketName,
483493
Key: destObjName,
@@ -496,7 +506,7 @@ describe('Object Part Copy', () => {
496506
it('should overwrite an existing part by copying a part', () => {
497507
// AWS response etag for this completed MPU
498508
const finalObjETag = '"db77ebbae9e9f5a244a26b86193ad818-1"';
499-
process.stdout.write('Putting first part in MPU test');
509+
process.stdout.write('Putting first part in MPU test\n');
500510
return s3.send(new UploadPartCopyCommand({
501511
Bucket: destBucketName,
502512
Key: destObjName,
@@ -507,7 +517,7 @@ describe('Object Part Copy', () => {
507517
assert.strictEqual(res.CopyPartResult.ETag, totalMpuObjectHash);
508518
assert(res.CopyPartResult.LastModified);
509519
}).then(() => {
510-
process.stdout.write('Overwriting first part in MPU test');
520+
process.stdout.write('Overwriting first part in MPU test\n');
511521
return s3.send(new UploadPartCopyCommand({
512522
Bucket: destBucketName,
513523
Key: destObjName,
@@ -518,7 +528,7 @@ describe('Object Part Copy', () => {
518528
).then(res => {
519529
assert.strictEqual(res.CopyPartResult.ETag, etag);
520530
assert(res.CopyPartResult.LastModified);
521-
process.stdout.write('Completing MPU');
531+
process.stdout.write('Completing MPU\n');
522532
return s3.send(new CompleteMultipartUploadCommand({
523533
Bucket: destBucketName,
524534
Key: destObjName,
@@ -535,7 +545,7 @@ describe('Object Part Copy', () => {
535545
assert.strictEqual(res.ETag, finalObjETag);
536546
}).then(() => {
537547
process.stdout.write('Getting object put by MPU with ' +
538-
'overwrite part');
548+
'overwrite part\n');
539549
return s3.send(new GetObjectCommand({
540550
Bucket: destBucketName,
541551
Key: destObjName,
@@ -552,7 +562,7 @@ describe('Object Part Copy', () => {
552562
it('should not corrupt object if overwriting an existing part by copying a part ' +
553563
'while the MPU is being completed', async () => {
554564
const finalObjETag = '"db77ebbae9e9f5a244a26b86193ad818-1"';
555-
process.stdout.write('Putting first part in MPU test"');
565+
process.stdout.write('Putting first part in MPU test\n');
556566
const randomDestObjName = `copycatobject${Math.floor(Math.random() * 100000)}`;
557567

558568
const initiateRes = await s3.send(new CreateMultipartUploadCommand({
@@ -572,7 +582,7 @@ describe('Object Part Copy', () => {
572582
assert(res.CopyPartResult.LastModified);
573583

574584
process.stdout.write(
575-
'Overwriting first part in MPU test and completing MPU at the same time',
585+
'Overwriting first part in MPU test and completing MPU at the same time\n',
576586
);
577587
const [completeRes, uploadRes] = await Promise.all([
578588
s3.send(new CompleteMultipartUploadCommand({
@@ -611,7 +621,7 @@ describe('Object Part Copy', () => {
611621
assert.strictEqual(completeRes.Key, randomDestObjName);
612622
assert.strictEqual(completeRes.ETag, finalObjETag);
613623
process.stdout.write(
614-
'Getting object put by MPU with ' + 'overwrite part',
624+
'Getting object put by MPU with overwrite part\n',
615625
);
616626
const resGet = await s3
617627
.send(new GetObjectCommand({
@@ -749,15 +759,15 @@ describe('Object Part Copy', () => {
749759
let otherAccountUploadId;
750760

751761
beforeEach(() => {
752-
process.stdout.write('In other account before each');
762+
process.stdout.write('In other account before each\n');
753763
return otherAccountS3.send(new CreateBucketCommand({ Bucket:
754764
otherAccountBucket }))
755765
.catch(err => {
756766
process.stdout.write('Error creating other account ' +
757767
`bucket: ${err}\n`);
758768
throw err;
759769
}).then(() => {
760-
process.stdout.write('Initiating other account MPU');
770+
process.stdout.write('Initiating other account MPU\n');
761771
return otherAccountS3.send(new CreateMultipartUploadCommand({
762772
Bucket: otherAccountBucket,
763773
Key: otherAccountKey,

0 commit comments

Comments
 (0)