Skip to content

Commit 3f3d671

Browse files
bert-ebert-e
committed
Merge branch 'bugfix/CLDSRV-835-s3c-teardown' into tmp/octopus/w/9.4/bugfix/CLDSRV-835-s3c-teardown
2 parents 3db6c18 + 3a5f77a commit 3f3d671

File tree

5 files changed

+133
-172
lines changed

5 files changed

+133
-172
lines changed

‎tests/functional/aws-node-sdk/lib/utility/bucket-util.js‎

Lines changed: 54 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ const {
44
CreateBucketCommand,
55
DeleteBucketCommand,
66
ListObjectVersionsCommand,
7+
DeleteObjectsCommand,
78
DeleteObjectCommand,
89
ListBucketsCommand,
910
} = require('@aws-sdk/client-s3');
@@ -90,48 +91,65 @@ class BucketUtility {
9091
* @param bucketName
9192
* @returns {Promise.<T>}
9293
*/
93-
empty(bucketName, BypassGovernanceRetention = false) {
94-
const param = {
95-
Bucket: bucketName,
96-
};
97-
98-
return this.s3.send(new ListObjectVersionsCommand(param))
99-
.then(data => Promise.all(
100-
(data.Versions || [])
101-
.filter(object => !object.Key.endsWith('/'))
102-
.map(object =>
103-
this.s3.send(new DeleteObjectCommand({
94+
async empty(bucketName, BypassGovernanceRetention = false) {
95+
let keyMarker = undefined;
96+
let versionIdMarker = undefined;
97+
let isTruncated = true;
98+
99+
while (isTruncated) {
100+
const response = await this.s3.send(new ListObjectVersionsCommand({
101+
Bucket: bucketName,
102+
KeyMarker: keyMarker,
103+
VersionIdMarker: versionIdMarker,
104+
}));
105+
106+
const objects = [
107+
...(response.Versions || []),
108+
...(response.DeleteMarkers || []),
109+
].map(({ Key, VersionId }) => ({ Key, VersionId }));
110+
111+
if (objects.length > 0) {
112+
try {
113+
const result = await this.s3.send(new DeleteObjectsCommand({
104114
Bucket: bucketName,
105-
Key: object.Key,
106-
VersionId: object.VersionId,
115+
Delete: {
116+
Objects: objects,
117+
Quiet: true
118+
},
107119
...(BypassGovernanceRetention && { BypassGovernanceRetention }),
108-
})).then(() => object)
109-
)
110-
.concat((data.Versions || [])
111-
.filter(object => object.Key.endsWith('/'))
112-
.map(object =>
120+
}));
121+
if (result.Errors && result.Errors.length > 0) {
122+
for (const e of result.Errors) {
123+
// eslint-disable-next-line no-console
124+
console.warn(
125+
`Warning BucketUtility.empty(): failed to delete s3://${bucketName}/${e.Key}` +
126+
` (versionId=${e.VersionId}): ${e.Code} - ${e.Message}`
127+
);
128+
}
129+
}
130+
} catch (err) {
131+
// Older cloudserver versions reject DeleteObjects with BadDigest
132+
// due to a Content-MD5 integrity check issue. Fall back to individual deletes.
133+
if (err.name !== 'BadDigest') {
134+
throw err;
135+
}
136+
await Promise.all(objects.map(({ Key, VersionId }) =>
113137
this.s3.send(new DeleteObjectCommand({
114138
Bucket: bucketName,
115-
Key: object.Key,
116-
VersionId: object.VersionId,
139+
Key,
140+
VersionId,
117141
...(BypassGovernanceRetention && { BypassGovernanceRetention }),
118142
}))
119-
.then(() => object)
120-
)
121-
)
122-
.concat((data.DeleteMarkers || [])
123-
.map(object =>
124-
this.s3.send(new DeleteObjectCommand({
125-
Bucket: bucketName,
126-
Key: object.Key,
127-
VersionId: object.VersionId,
128-
...(BypassGovernanceRetention && { BypassGovernanceRetention }),
129-
}))
130-
.then(() => object)
131-
)
132-
)
133-
)
134-
);
143+
));
144+
}
145+
}
146+
147+
isTruncated = response.IsTruncated;
148+
if (isTruncated) {
149+
keyMarker = response.NextKeyMarker;
150+
versionIdMarker = response.NextVersionIdMarker;
151+
}
152+
}
135153
}
136154

137155
emptyMany(bucketNames) {

‎tests/functional/aws-node-sdk/lib/utility/provideRawOutput.js‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
const cp = require('child_process');
2+
const util = require('util');
23

34
const conf = require('../../../../../lib/Config').config;
45

@@ -44,4 +45,9 @@ function provideRawOutput(args, cb) {
4445
});
4546
}
4647

48+
provideRawOutput[util.promisify.custom] = args =>
49+
new Promise(resolve =>
50+
provideRawOutput(args, (httpCode, rawOutput) => resolve({ httpCode, rawOutput }))
51+
);
52+
4753
module.exports = provideRawOutput;

‎tests/functional/aws-node-sdk/test/legacy/authV2QueryTests.js‎

Lines changed: 22 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
const assert = require('assert');
22
const process = require('node:process');
33
const cp = require('child_process');
4+
const util = require('util');
5+
const timers = require('timers/promises');
46
const {
57
S3Client,
68
CreateBucketCommand,
@@ -12,6 +14,7 @@ const {
1214
const { getSignedUrl } = require('@aws-sdk/s3-request-presigner');
1315
const getConfig = require('../support/config');
1416
const provideRawOutput = require('../../lib/utility/provideRawOutput');
17+
const provideRawOutputAsync = util.promisify(provideRawOutput);
1518

1619
const random = Math.round(Math.random() * 100).toString();
1720
const bucket = `mybucket-${random}`;
@@ -60,49 +63,32 @@ describe('aws-node-sdk v2auth query tests', function testSuite() {
6063
const futureExpiry = Math.floor(Date.now() / 1000) + 604810; // 10 seconds more than limit
6164
urlObj.searchParams.set('Expires', futureExpiry.toString());
6265
const invalidUrl = urlObj.toString();
63-
await new Promise(resolve => {
64-
provideRawOutput(['-verbose', '-X', 'PUT', invalidUrl], httpCode => {
65-
assert.strictEqual(httpCode, '403 FORBIDDEN');
66-
resolve();
67-
});
68-
});
66+
const { httpCode } = await provideRawOutputAsync(['-verbose', '-X', 'PUT', invalidUrl]);
67+
assert.strictEqual(httpCode, '403 FORBIDDEN');
6968
});
7069

7170
it('should return an error code if request occurs after expiry', async () => {
7271
const command = new CreateBucketCommand({ Bucket: bucket });
7372
const url = await getSignedUrl(s3, command, { expiresIn: 1 });
74-
await new Promise(resolve => {
75-
setTimeout(() => {
76-
provideRawOutput(['-verbose', '-X', 'PUT', url], httpCode => {
77-
assert.strictEqual(httpCode, '403 FORBIDDEN');
78-
resolve();
79-
});
80-
}, 1500);
81-
});
73+
await timers.setTimeout(1500);
74+
const { httpCode } = await provideRawOutputAsync(['-verbose', '-X', 'PUT', url]);
75+
assert.strictEqual(httpCode, '403 FORBIDDEN');
8276
});
8377

8478
it('should create a bucket', async () => {
8579
const command = new CreateBucketCommand({ Bucket: bucket });
8680
const url = await getSignedUrl(s3, command, { expiresIn: almostOutsideTime });
87-
await new Promise(resolve => {
88-
provideRawOutput(['-verbose', '-X', 'PUT', url], httpCode => {
89-
assert.strictEqual(httpCode, '200 OK');
90-
resolve();
91-
});
92-
});
81+
const { httpCode } = await provideRawOutputAsync(['-verbose', '-X', 'PUT', url]);
82+
assert.strictEqual(httpCode, '200 OK');
9383
});
9484

9585

9686
it('should put an object', async () => {
9787
const command = new PutObjectCommand({ Bucket: bucket, Key: 'key' });
9888
const url = await getSignedUrl(s3, command, { expiresIn: almostOutsideTime });
99-
await new Promise(resolve => {
100-
provideRawOutput(['-verbose', '-X', 'PUT', url,
101-
'--upload-file', 'uploadFile'], httpCode => {
102-
assert.strictEqual(httpCode, '200 OK');
103-
resolve();
104-
});
105-
});
89+
const { httpCode } = await provideRawOutputAsync(['-verbose', '-X', 'PUT', url,
90+
'--upload-file', 'uploadFile']);
91+
assert.strictEqual(httpCode, '200 OK');
10692
});
10793

10894
it('should put an object with an acl setting and a storage class setting', async () => {
@@ -118,25 +104,17 @@ describe('aws-node-sdk v2auth query tests', function testSuite() {
118104
StorageClass: 'STANDARD'
119105
});
120106
const url = await getSignedUrl(s3, command);
121-
await new Promise(resolve => {
122-
provideRawOutput(['-verbose', '-X', 'PUT', url,
123-
'--upload-file', 'uploadFile'], httpCode => {
124-
assert.strictEqual(httpCode, '200 OK');
125-
resolve();
126-
});
127-
});
107+
const { httpCode } = await provideRawOutputAsync(['-verbose', '-X', 'PUT', url,
108+
'--upload-file', 'uploadFile']);
109+
assert.strictEqual(httpCode, '200 OK');
128110
});
129111

130112

131113
it('should get an object', async () => {
132114
const command = new GetObjectCommand({ Bucket: bucket, Key: 'key' });
133115
const url = await getSignedUrl(s3, command, { expiresIn: almostOutsideTime });
134-
await new Promise(resolve => {
135-
provideRawOutput(['-verbose', '-o', 'download', url], httpCode => {
136-
assert.strictEqual(httpCode, '200 OK');
137-
resolve();
138-
});
139-
});
116+
const { httpCode } = await provideRawOutputAsync(['-verbose', '-o', 'download', url]);
117+
assert.strictEqual(httpCode, '200 OK');
140118
});
141119

142120
it('downloaded file should equal file that was put', done => {
@@ -148,23 +126,15 @@ describe('aws-node-sdk v2auth query tests', function testSuite() {
148126
it('should delete an object', async () => {
149127
const command = new DeleteObjectCommand({ Bucket: bucket, Key: 'key' });
150128
const url = await getSignedUrl(s3, command, { expiresIn: almostOutsideTime });
151-
await new Promise(resolve => {
152-
provideRawOutput(['-verbose', '-X', 'DELETE', url], httpCode => {
153-
assert.strictEqual(httpCode, '204 NO CONTENT');
154-
resolve();
155-
});
156-
});
129+
const { httpCode } = await provideRawOutputAsync(['-verbose', '-X', 'DELETE', url]);
130+
assert.strictEqual(httpCode, '204 NO CONTENT');
157131
});
158132

159133

160134
it('should delete a bucket', async () => {
161135
const command = new DeleteBucketCommand({ Bucket: bucket });
162136
const url = await getSignedUrl(s3, command, { expiresIn: almostOutsideTime });
163-
await new Promise(resolve => {
164-
provideRawOutput(['-verbose', '-X', 'DELETE', url], httpCode => {
165-
assert.strictEqual(httpCode, '204 NO CONTENT');
166-
resolve();
167-
});
168-
});
137+
const { httpCode } = await provideRawOutputAsync(['-verbose', '-X', 'DELETE', url]);
138+
assert.strictEqual(httpCode, '204 NO CONTENT');
169139
});
170140
});

0 commit comments

Comments
 (0)