Skip to content

Commit 57632de

Browse files
jonathan-gramainjonathan-gramain
committed
Merge remote-tracking branch 'origin/bugfix/CLDSRV-647-exceptionOnPutIfNoReplicationEndpoint' into w/7.70/bugfix/CLDSRV-647-exceptionOnPutIfNoReplicationEndpoint
2 parents d1bcff9 + 6edbf9c commit 57632de

10 files changed

Lines changed: 140 additions & 29 deletions

File tree

‎lib/api/apiUtils/object/createAndStoreObject.js‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,8 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
136136
size,
137137
headers,
138138
isDeleteMarker,
139-
replicationInfo: getReplicationInfo(objectKey, bucketMD, false, size, null, null, authInfo, isDeleteMarker),
139+
replicationInfo: getReplicationInfo(config,
140+
objectKey, bucketMD, false, size, null, null, authInfo, isDeleteMarker),
140141
overheadField,
141142
log,
142143
};

‎lib/api/apiUtils/object/getReplicationInfo.js‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
const s3config = require('../../../Config').config;
21
const { isLifecycleSession } = require('../authorization/permissionChecks.js');
32

43
function _getBackend(objectMD, site) {
@@ -15,7 +14,7 @@ function _getBackend(objectMD, site) {
1514
};
1615
}
1716

18-
function _getStorageClasses(rule) {
17+
function _getStorageClasses(s3config, rule) {
1918
if (rule.storageClass) {
2019
return rule.storageClass.split(',');
2120
}
@@ -26,14 +25,20 @@ function _getStorageClasses(rule) {
2625
replicationEndpoints.find(endpoint => endpoint.default);
2726
return [endPoint.site];
2827
}
28+
if (replicationEndpoints.length === 0) {
29+
return [];
30+
}
2931
return [replicationEndpoints[0].site];
3032
}
3133

32-
function _getReplicationInfo(rule, replicationConfig, content, operationType,
34+
function _getReplicationInfo(s3config, rule, replicationConfig, content, operationType,
3335
objectMD) {
3436
const storageTypes = [];
3537
const backends = [];
36-
const storageClasses = _getStorageClasses(rule);
38+
const storageClasses = _getStorageClasses(s3config, rule);
39+
if (storageClasses.length === 0) {
40+
return undefined;
41+
}
3742
storageClasses.forEach(storageClass => {
3843
const location = s3config.locationConstraints[storageClass];
3944
if (location && ['aws_s3', 'azure'].includes(location.type)) {
@@ -58,6 +63,9 @@ function _getReplicationInfo(rule, replicationConfig, content, operationType,
5863
/**
5964
* Get the object replicationInfo to replicate data and metadata, or only
6065
* metadata if the operation only changes metadata or the object is 0 bytes
66+
* @param {object} s3config - Cloudserver configuration object
67+
* @param {object} s3config.locationConstraints - Configured map of location constraints
68+
* @param {object[]} s3config.replicationEndpoints - Configured replication endpoints
6169
* @param {string} objKey - The key of the object
6270
* @param {object} bucketMD - The bucket metadata
6371
* @param {boolean} isMD - Whether the operation is only updating metadata
@@ -68,8 +76,8 @@ function _getReplicationInfo(rule, replicationConfig, content, operationType,
6876
* @param {boolean} [isDeleteMarker] - whether creating a delete marker
6977
* @return {undefined}
7078
*/
71-
function getReplicationInfo(objKey, bucketMD, isMD, objSize, operationType,
72-
objectMD, authInfo, isDeleteMarker) {
79+
function getReplicationInfo(s3config,
80+
objKey, bucketMD, isMD, objSize, operationType, objectMD, authInfo, isDeleteMarker) {
7381
const content = isMD || objSize === 0 ? ['METADATA'] : ['DATA', 'METADATA'];
7482
const config = bucketMD.getReplicationConfiguration();
7583
// If bucket does not have a replication configuration, do not replicate.
@@ -83,7 +91,7 @@ function getReplicationInfo(objKey, bucketMD, isMD, objSize, operationType,
8391
const rule = config.rules.find(rule =>
8492
(objKey.startsWith(rule.prefix) && rule.enabled));
8593
if (rule) {
86-
return _getReplicationInfo(rule, config, content, operationType,
94+
return _getReplicationInfo(s3config, rule, config, content, operationType,
8795
objectMD);
8896
}
8997
}

‎lib/api/completeMultipartUpload.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -301,8 +301,8 @@ function completeMultipartUpload(authInfo, request, log, callback) {
301301
contentMD5: aggregateETag,
302302
size: calculatedSize,
303303
multipart: true,
304-
replicationInfo: getReplicationInfo(objectKey, destBucket,
305-
false, calculatedSize, REPLICATION_ACTION),
304+
replicationInfo: getReplicationInfo(config,
305+
objectKey, destBucket, false, calculatedSize, REPLICATION_ACTION),
306306
originOp: 's3:ObjectCreated:CompleteMultipartUpload',
307307
overheadField: constants.overheadField,
308308
log,

‎lib/api/objectCopy.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -168,8 +168,8 @@ function _prepMetadata(request, sourceObjMD, headers, sourceIsDestination,
168168
lastModifiedDate: new Date().toJSON(),
169169
tagging,
170170
taggingCopy,
171-
replicationInfo: getReplicationInfo(objectKey, destBucketMD, false,
172-
sourceObjMD['content-length']),
171+
replicationInfo: getReplicationInfo(config,
172+
objectKey, destBucketMD, false, sourceObjMD['content-length']),
173173
locationMatch,
174174
originOp: 's3:ObjectCreated:Copy',
175175
};

‎lib/api/objectDeleteTagging.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,8 +76,8 @@ function objectDeleteTagging(authInfo, request, log, callback) {
7676
// eslint-disable-next-line no-param-reassign
7777
objectMD.tags = {};
7878
const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode);
79-
const replicationInfo = getReplicationInfo(objectKey, bucket, true,
80-
0, REPLICATION_ACTION, objectMD);
79+
const replicationInfo = getReplicationInfo(config,
80+
objectKey, bucket, true, 0, REPLICATION_ACTION, objectMD);
8181
if (replicationInfo) {
8282
// eslint-disable-next-line no-param-reassign
8383
objectMD.replicationInfo = Object.assign({},

‎lib/api/objectPutLegalHold.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -87,8 +87,8 @@ function objectPutLegalHold(authInfo, request, log, callback) {
8787
// eslint-disable-next-line no-param-reassign
8888
objectMD.legalHold = legalHold;
8989
const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode);
90-
const replicationInfo = getReplicationInfo(objectKey, bucket, true,
91-
0, REPLICATION_ACTION, objectMD);
90+
const replicationInfo = getReplicationInfo(config,
91+
objectKey, bucket, true, 0, REPLICATION_ACTION, objectMD);
9292
if (replicationInfo) {
9393
// eslint-disable-next-line no-param-reassign
9494
objectMD.replicationInfo = Object.assign({},

‎lib/api/objectPutRetention.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -125,8 +125,8 @@ function objectPutRetention(authInfo, request, log, callback) {
125125
objectMD.retentionMode = retentionInfo.mode;
126126
objectMD.retentionDate = retentionInfo.date;
127127
const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode);
128-
const replicationInfo = getReplicationInfo(objectKey, bucket, true,
129-
0, REPLICATION_ACTION, objectMD);
128+
const replicationInfo = getReplicationInfo(config,
129+
objectKey, bucket, true, 0, REPLICATION_ACTION, objectMD);
130130
if (replicationInfo) {
131131
objectMD.replicationInfo = Object.assign({},
132132
objectMD.replicationInfo, replicationInfo);

‎lib/api/objectPutTagging.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -81,8 +81,8 @@ function objectPutTagging(authInfo, request, log, callback) {
8181
// eslint-disable-next-line no-param-reassign
8282
objectMD.tags = tags;
8383
const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode);
84-
const replicationInfo = getReplicationInfo(objectKey, bucket, true,
85-
0, REPLICATION_ACTION, objectMD);
84+
const replicationInfo = getReplicationInfo(config,
85+
objectKey, bucket, true, 0, REPLICATION_ACTION, objectMD);
8686
if (replicationInfo) {
8787
// eslint-disable-next-line no-param-reassign
8888
objectMD.replicationInfo = Object.assign({},

‎lib/metadata/acl.js‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ const aclUtils = require('../utilities/aclUtils');
55
const constants = require('../../constants');
66
const metadata = require('../metadata/wrapper');
77
const vault = require('../auth/vault');
8+
const { config } = require('../Config');
89

910
const acl = {
1011
addACL(bucket, addACLParams, log, cb) {
@@ -19,7 +20,7 @@ const acl = {
1920
objectMD.acl = addACLParams;
2021
// eslint-disable-next-line no-param-reassign
2122
objectMD.originOp = 's3:ObjectAcl:Put';
22-
const replicationInfo = getReplicationInfo(objectKey, bucket, true);
23+
const replicationInfo = getReplicationInfo(config, objectKey, bucket, true);
2324
if (replicationInfo) {
2425
// eslint-disable-next-line no-param-reassign
2526
objectMD.replicationInfo = Object.assign({},

‎tests/unit/api/apiUtils/getReplicationInfo.js‎

Lines changed: 109 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,39 @@ const getReplicationInfo =
55
require('../../../../lib/api/apiUtils/object/getReplicationInfo');
66
const { makeAuthInfo } = require('../../helpers');
77

8-
function _getObjectReplicationInfo(replicationConfig, authInfo, isDeleteMarker) {
8+
function _getObjectReplicationInfo(replicationEndpoints, replicationConfig, authInfo, isDeleteMarker) {
99
const bucketInfo = new BucketInfo(
1010
'testbucket', 'someCanonicalId', 'accountDisplayName',
1111
new Date().toJSON(),
1212
null, null, null, null, null, null, null, null, null,
1313
replicationConfig);
14-
return getReplicationInfo('fookey', bucketInfo, true, 123, null, null, authInfo, isDeleteMarker);
14+
return getReplicationInfo(replicationEndpoints,
15+
'fookey', bucketInfo, true, 123, null, null, authInfo, isDeleteMarker);
1516
}
1617

18+
const TEST_CONFIG = {
19+
locationConstraints: {
20+
awsbackend: {
21+
type: 'aws_s3',
22+
legacyAwsBehavior: true,
23+
details: {
24+
awsEndpoint: 's3.amazonaws.com',
25+
bucketName: 'multitester555',
26+
bucketMatch: true,
27+
credentialsProfile: 'default',
28+
},
29+
},
30+
},
31+
replicationEndpoints: [{
32+
site: 'zenko',
33+
servers: ['127.0.0.1:8000'],
34+
default: true,
35+
}, {
36+
site: 'us-east-2',
37+
type: 'aws_s3',
38+
}],
39+
};
40+
1741
describe('getReplicationInfo helper', () => {
1842
it('should get replication info when rules are enabled', () => {
1943
const replicationConfig = {
@@ -25,7 +49,7 @@ describe('getReplicationInfo helper', () => {
2549
}],
2650
destination: 'tosomewhere',
2751
};
28-
const replicationInfo = _getObjectReplicationInfo(replicationConfig);
52+
const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig);
2953
assert.deepStrictEqual(replicationInfo, {
3054
status: 'PENDING',
3155
backends: [{
@@ -53,7 +77,8 @@ describe('getReplicationInfo helper', () => {
5377
};
5478

5579
const authInfo = makeAuthInfo('accessKey1', null, 'another-session');
56-
const replicationInfo = _getObjectReplicationInfo(replicationConfig, authInfo, true);
80+
const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG,
81+
replicationConfig, authInfo, true);
5782

5883
assert.deepStrictEqual(replicationInfo, {
5984
status: 'PENDING',
@@ -83,7 +108,8 @@ describe('getReplicationInfo helper', () => {
83108
};
84109

85110
const authInfo = makeAuthInfo('accessKey1', null, 'backbeat-lifecycle');
86-
const replicationInfo = _getObjectReplicationInfo(replicationConfig, authInfo, false);
111+
const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG,
112+
replicationConfig, authInfo, false);
87113

88114
assert.deepStrictEqual(replicationInfo, {
89115
status: 'PENDING',
@@ -110,11 +136,11 @@ describe('getReplicationInfo helper', () => {
110136
}],
111137
destination: 'tosomewhere',
112138
};
113-
const replicationInfo = _getObjectReplicationInfo(replicationConfig);
139+
const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig);
114140
assert.deepStrictEqual(replicationInfo, undefined);
115141
});
116142

117-
it('should not get replication info when action comming from lifecycle session', () => {
143+
it('should not get replication info when action coming from lifecycle session', () => {
118144
const replicationConfig = {
119145
role: 'arn:aws:iam::root:role/s3-replication-role',
120146
rules: [{
@@ -126,8 +152,83 @@ describe('getReplicationInfo helper', () => {
126152
};
127153

128154
const authInfo = makeAuthInfo('accessKey1', null, 'backbeat-lifecycle');
129-
const replicationInfo = _getObjectReplicationInfo(replicationConfig, authInfo, true);
155+
const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG,
156+
replicationConfig, authInfo, true);
157+
158+
assert.deepStrictEqual(replicationInfo, undefined);
159+
});
160+
161+
it('should get replication info with default StorageClass when rules are enabled', () => {
162+
const replicationConfig = {
163+
role: 'arn:aws:iam::root:role/s3-replication-role-1,arn:aws:iam::root:role/s3-replication-role-2',
164+
rules: [{
165+
prefix: '',
166+
enabled: true,
167+
}],
168+
destination: 'tosomewhere',
169+
};
170+
const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig);
171+
assert.deepStrictEqual(replicationInfo, {
172+
status: 'PENDING',
173+
backends: [{
174+
site: 'zenko',
175+
status: 'PENDING',
176+
dataStoreVersionId: '',
177+
}],
178+
content: ['METADATA'],
179+
destination: 'tosomewhere',
180+
storageClass: 'zenko',
181+
role: 'arn:aws:iam::root:role/s3-replication-role-1,arn:aws:iam::root:role/s3-replication-role-2',
182+
storageType: '',
183+
});
184+
});
130185

186+
it('should return undefined with specified StorageClass mode if no replication endpoint is configured', () => {
187+
const replicationConfig = {
188+
role: 'arn:aws:iam::root:role/s3-replication-role',
189+
rules: [{
190+
prefix: '',
191+
enabled: true,
192+
storageClass: 'awsbackend',
193+
}],
194+
destination: 'tosomewhere',
195+
};
196+
const configWithNoReplicationEndpoint = {
197+
locationConstraints: TEST_CONFIG.locationConstraints,
198+
replicationEndpoints: [],
199+
};
200+
const replicationInfo = _getObjectReplicationInfo(configWithNoReplicationEndpoint,
201+
replicationConfig);
202+
assert.deepStrictEqual(replicationInfo, {
203+
status: 'PENDING',
204+
backends: [{
205+
site: 'awsbackend',
206+
status: 'PENDING',
207+
dataStoreVersionId: '',
208+
}],
209+
content: ['METADATA'],
210+
destination: 'tosomewhere',
211+
storageClass: 'awsbackend',
212+
role: 'arn:aws:iam::root:role/s3-replication-role',
213+
storageType: 'aws_s3',
214+
});
215+
});
216+
217+
it('should return undefined with default StorageClass if no replication endpoint is configured', () => {
218+
const replicationConfig = {
219+
role: 'arn:aws:iam::root:role/s3-replication-role-1,arn:aws:iam::root:role/s3-replication-role-2',
220+
rules: [{
221+
prefix: '',
222+
enabled: true,
223+
}],
224+
destination: 'tosomewhere',
225+
};
226+
const configWithNoReplicationEndpoint = {
227+
locationConstraints: TEST_CONFIG.locationConstraints,
228+
replicationEndpoints: [],
229+
};
230+
const replicationInfo = _getObjectReplicationInfo(configWithNoReplicationEndpoint,
231+
replicationConfig);
131232
assert.deepStrictEqual(replicationInfo, undefined);
132233
});
133234
});

0 commit comments

Comments
 (0)