Skip to content

Commit 787728a

Browse files
jonathan-gramainjonathan-gramain
committed
CLDSRV-647 [refactor] pass S3 config to getReplicationInfo
Instead of pulling the location constraints and replication endpoints from the Cloudserver config within the getReplicationInfo helper itself, pass the config object explicitly. This makes the function pure, using only its input parameters, hence more test cases can be easily added with a different configuration for location constraints and/or replication endpoints.
1 parent 5c4a1b3 commit 787728a

10 files changed

Lines changed: 64 additions & 29 deletions

File tree

‎lib/api/apiUtils/object/createAndStoreObject.js‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,8 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
135135
size,
136136
headers,
137137
isDeleteMarker,
138-
replicationInfo: getReplicationInfo(objectKey, bucketMD, false, size, null, null, authInfo, isDeleteMarker),
138+
replicationInfo: getReplicationInfo(config,
139+
objectKey, bucketMD, false, size, null, null, authInfo, isDeleteMarker),
139140
log,
140141
};
141142

‎lib/api/apiUtils/object/getReplicationInfo.js‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
const s3config = require('../../../Config').config;
21
const { isLifecycleSession } = require('../authorization/permissionChecks.js');
32

43
function _getBackend(objectMD, site) {
@@ -15,7 +14,7 @@ function _getBackend(objectMD, site) {
1514
};
1615
}
1716

18-
function _getStorageClasses(rule) {
17+
function _getStorageClasses(s3config, rule) {
1918
if (rule.storageClass) {
2019
return rule.storageClass.split(',');
2120
}
@@ -29,11 +28,11 @@ function _getStorageClasses(rule) {
2928
return [replicationEndpoints[0].site];
3029
}
3130

32-
function _getReplicationInfo(rule, replicationConfig, content, operationType,
31+
function _getReplicationInfo(s3config, rule, replicationConfig, content, operationType,
3332
objectMD) {
3433
const storageTypes = [];
3534
const backends = [];
36-
const storageClasses = _getStorageClasses(rule);
35+
const storageClasses = _getStorageClasses(s3config, rule);
3736
storageClasses.forEach(storageClass => {
3837
const location = s3config.locationConstraints[storageClass];
3938
if (location && ['aws_s3', 'azure'].includes(location.type)) {
@@ -58,6 +57,9 @@ function _getReplicationInfo(rule, replicationConfig, content, operationType,
5857
/**
5958
* Get the object replicationInfo to replicate data and metadata, or only
6059
* metadata if the operation only changes metadata or the object is 0 bytes
60+
* @param {object} s3config - Cloudserver configuration object
61+
* @param {object} s3config.locationConstraints - Configured map of location constraints
62+
* @param {object[]} s3config.replicationEndpoints - Configured replication endpoints
6163
* @param {string} objKey - The key of the object
6264
* @param {object} bucketMD - The bucket metadata
6365
* @param {boolean} isMD - Whether the operation is only updating metadata
@@ -68,8 +70,8 @@ function _getReplicationInfo(rule, replicationConfig, content, operationType,
6870
* @param {boolean} [isDeleteMarker] - whether creating a delete marker
6971
* @return {undefined}
7072
*/
71-
function getReplicationInfo(objKey, bucketMD, isMD, objSize, operationType,
72-
objectMD, authInfo, isDeleteMarker) {
73+
function getReplicationInfo(s3config,
74+
objKey, bucketMD, isMD, objSize, operationType, objectMD, authInfo, isDeleteMarker) {
7375
const content = isMD || objSize === 0 ? ['METADATA'] : ['DATA', 'METADATA'];
7476
const config = bucketMD.getReplicationConfiguration();
7577
// If bucket does not have a replication configuration, do not replicate.
@@ -83,7 +85,7 @@ function getReplicationInfo(objKey, bucketMD, isMD, objSize, operationType,
8385
const rule = config.rules.find(rule =>
8486
(objKey.startsWith(rule.prefix) && rule.enabled));
8587
if (rule) {
86-
return _getReplicationInfo(rule, config, content, operationType,
88+
return _getReplicationInfo(s3config, rule, config, content, operationType,
8789
objectMD);
8890
}
8991
}

‎lib/api/completeMultipartUpload.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -301,8 +301,8 @@ function completeMultipartUpload(authInfo, request, log, callback) {
301301
contentMD5: aggregateETag,
302302
size: calculatedSize,
303303
multipart: true,
304-
replicationInfo: getReplicationInfo(objectKey, destBucket,
305-
false, calculatedSize, REPLICATION_ACTION),
304+
replicationInfo: getReplicationInfo(config,
305+
objectKey, destBucket, false, calculatedSize, REPLICATION_ACTION),
306306
originOp: 's3:ObjectCreated:CompleteMultipartUpload',
307307
log,
308308
};

‎lib/api/objectCopy.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -167,8 +167,8 @@ function _prepMetadata(request, sourceObjMD, headers, sourceIsDestination,
167167
lastModifiedDate: new Date().toJSON(),
168168
tagging,
169169
taggingCopy,
170-
replicationInfo: getReplicationInfo(objectKey, destBucketMD, false,
171-
sourceObjMD['content-length']),
170+
replicationInfo: getReplicationInfo(config,
171+
objectKey, destBucketMD, false, sourceObjMD['content-length']),
172172
locationMatch,
173173
originOp: 's3:ObjectCreated:Copy',
174174
};

‎lib/api/objectDeleteTagging.js‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ const collectCorsHeaders = require('../utilities/collectCorsHeaders');
1010
const metadata = require('../metadata/wrapper');
1111
const getReplicationInfo = require('./apiUtils/object/getReplicationInfo');
1212
const { data } = require('../data/wrapper');
13+
const { config } = require('../Config');
1314
const REPLICATION_ACTION = 'DELETE_TAGGING';
1415

1516
/**
@@ -72,8 +73,8 @@ function objectDeleteTagging(authInfo, request, log, callback) {
7273
objectMD.tags = {};
7374
const params = objectMD.versionId ? { versionId:
7475
objectMD.versionId } : {};
75-
const replicationInfo = getReplicationInfo(objectKey, bucket, true,
76-
0, REPLICATION_ACTION, objectMD);
76+
const replicationInfo = getReplicationInfo(config,
77+
objectKey, bucket, true, 0, REPLICATION_ACTION, objectMD);
7778
if (replicationInfo) {
7879
// eslint-disable-next-line no-param-reassign
7980
objectMD.replicationInfo = Object.assign({},

‎lib/api/objectPutLegalHold.js‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ const getReplicationInfo = require('./apiUtils/object/getReplicationInfo');
88
const metadata = require('../metadata/wrapper');
99
const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils');
1010
const { pushMetric } = require('../utapi/utilities');
11+
const { config } = require('../Config');
1112

1213
const { parseLegalHoldXml } = s3middleware.objectLegalHold;
1314

@@ -84,8 +85,8 @@ function objectPutLegalHold(authInfo, request, log, callback) {
8485
objectMD.legalHold = legalHold;
8586
const params = objectMD.versionId ?
8687
{ versionId: objectMD.versionId } : {};
87-
const replicationInfo = getReplicationInfo(objectKey, bucket, true,
88-
0, REPLICATION_ACTION, objectMD);
88+
const replicationInfo = getReplicationInfo(config,
89+
objectKey, bucket, true, 0, REPLICATION_ACTION, objectMD);
8990
if (replicationInfo) {
9091
// eslint-disable-next-line no-param-reassign
9192
objectMD.replicationInfo = Object.assign({},

‎lib/api/objectPutRetention.js‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ const { pushMetric } = require('../utapi/utilities');
1010
const getReplicationInfo = require('./apiUtils/object/getReplicationInfo');
1111
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
1212
const metadata = require('../metadata/wrapper');
13+
const { config } = require('../Config');
1314

1415
const { parseRetentionXml } = s3middleware.retention;
1516
const REPLICATION_ACTION = 'PUT_RETENTION';
@@ -124,8 +125,8 @@ function objectPutRetention(authInfo, request, log, callback) {
124125
objectMD.retentionDate = retentionInfo.date;
125126
const params = objectMD.versionId ?
126127
{ versionId: objectMD.versionId } : {};
127-
const replicationInfo = getReplicationInfo(objectKey, bucket, true,
128-
0, REPLICATION_ACTION, objectMD);
128+
const replicationInfo = getReplicationInfo(config,
129+
objectKey, bucket, true, 0, REPLICATION_ACTION, objectMD);
129130
if (replicationInfo) {
130131
objectMD.replicationInfo = Object.assign({},
131132
objectMD.replicationInfo, replicationInfo);

‎lib/api/objectPutTagging.js‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ const getReplicationInfo = require('./apiUtils/object/getReplicationInfo');
1010
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
1111
const metadata = require('../metadata/wrapper');
1212
const { data } = require('../data/wrapper');
13+
const { config } = require('../Config');
1314
const { parseTagXml } = s3middleware.tagging;
1415
const REPLICATION_ACTION = 'PUT_TAGGING';
1516

@@ -77,8 +78,8 @@ function objectPutTagging(authInfo, request, log, callback) {
7778
objectMD.tags = tags;
7879
const params = objectMD.versionId ? { versionId:
7980
objectMD.versionId } : {};
80-
const replicationInfo = getReplicationInfo(objectKey, bucket, true,
81-
0, REPLICATION_ACTION, objectMD);
81+
const replicationInfo = getReplicationInfo(config,
82+
objectKey, bucket, true, 0, REPLICATION_ACTION, objectMD);
8283
if (replicationInfo) {
8384
// eslint-disable-next-line no-param-reassign
8485
objectMD.replicationInfo = Object.assign({},

‎lib/metadata/acl.js‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ const aclUtils = require('../utilities/aclUtils');
55
const constants = require('../../constants');
66
const metadata = require('../metadata/wrapper');
77
const vault = require('../auth/vault');
8+
const { config } = require('../Config');
89

910
const acl = {
1011
addACL(bucket, addACLParams, log, cb) {
@@ -19,7 +20,7 @@ const acl = {
1920
objectMD.acl = addACLParams;
2021
// eslint-disable-next-line no-param-reassign
2122
objectMD.originOp = 's3:ObjectAcl:Put';
22-
const replicationInfo = getReplicationInfo(objectKey, bucket, true);
23+
const replicationInfo = getReplicationInfo(config, objectKey, bucket, true);
2324
if (replicationInfo) {
2425
// eslint-disable-next-line no-param-reassign
2526
objectMD.replicationInfo = Object.assign({},

‎tests/unit/api/apiUtils/getReplicationInfo.js‎

Lines changed: 35 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,39 @@ const getReplicationInfo =
55
require('../../../../lib/api/apiUtils/object/getReplicationInfo');
66
const { makeAuthInfo } = require('../../helpers');
77

8-
function _getObjectReplicationInfo(replicationConfig, authInfo, isDeleteMarker) {
8+
function _getObjectReplicationInfo(replicationEndpoints, replicationConfig, authInfo, isDeleteMarker) {
99
const bucketInfo = new BucketInfo(
1010
'testbucket', 'someCanonicalId', 'accountDisplayName',
1111
new Date().toJSON(),
1212
null, null, null, null, null, null, null, null, null,
1313
replicationConfig);
14-
return getReplicationInfo('fookey', bucketInfo, true, 123, null, null, authInfo, isDeleteMarker);
14+
return getReplicationInfo(replicationEndpoints,
15+
'fookey', bucketInfo, true, 123, null, null, authInfo, isDeleteMarker);
1516
}
1617

18+
const TEST_CONFIG = {
19+
locationConstraints: {
20+
awsbackend: {
21+
type: 'aws_s3',
22+
legacyAwsBehavior: true,
23+
details: {
24+
awsEndpoint: 's3.amazonaws.com',
25+
bucketName: 'multitester555',
26+
bucketMatch: true,
27+
credentialsProfile: 'default',
28+
},
29+
},
30+
},
31+
replicationEndpoints: [{
32+
site: 'zenko',
33+
servers: ['127.0.0.1:8000'],
34+
default: true,
35+
}, {
36+
site: 'us-east-2',
37+
type: 'aws_s3',
38+
}],
39+
};
40+
1741
describe('getReplicationInfo helper', () => {
1842
it('should get replication info when rules are enabled', () => {
1943
const replicationConfig = {
@@ -25,7 +49,7 @@ describe('getReplicationInfo helper', () => {
2549
}],
2650
destination: 'tosomewhere',
2751
};
28-
const replicationInfo = _getObjectReplicationInfo(replicationConfig);
52+
const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig);
2953
assert.deepStrictEqual(replicationInfo, {
3054
status: 'PENDING',
3155
backends: [{
@@ -53,7 +77,8 @@ describe('getReplicationInfo helper', () => {
5377
};
5478

5579
const authInfo = makeAuthInfo('accessKey1', null, 'another-session');
56-
const replicationInfo = _getObjectReplicationInfo(replicationConfig, authInfo, true);
80+
const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG,
81+
replicationConfig, authInfo, true);
5782

5883
assert.deepStrictEqual(replicationInfo, {
5984
status: 'PENDING',
@@ -83,7 +108,8 @@ describe('getReplicationInfo helper', () => {
83108
};
84109

85110
const authInfo = makeAuthInfo('accessKey1', null, 'backbeat-lifecycle');
86-
const replicationInfo = _getObjectReplicationInfo(replicationConfig, authInfo, false);
111+
const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG,
112+
replicationConfig, authInfo, false);
87113

88114
assert.deepStrictEqual(replicationInfo, {
89115
status: 'PENDING',
@@ -110,11 +136,11 @@ describe('getReplicationInfo helper', () => {
110136
}],
111137
destination: 'tosomewhere',
112138
};
113-
const replicationInfo = _getObjectReplicationInfo(replicationConfig);
139+
const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig);
114140
assert.deepStrictEqual(replicationInfo, undefined);
115141
});
116142

117-
it('should not get replication info when action comming from lifecycle session', () => {
143+
it('should not get replication info when action coming from lifecycle session', () => {
118144
const replicationConfig = {
119145
role: 'arn:aws:iam::root:role/s3-replication-role',
120146
rules: [{
@@ -126,7 +152,8 @@ describe('getReplicationInfo helper', () => {
126152
};
127153

128154
const authInfo = makeAuthInfo('accessKey1', null, 'backbeat-lifecycle');
129-
const replicationInfo = _getObjectReplicationInfo(replicationConfig, authInfo, true);
155+
const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG,
156+
replicationConfig, authInfo, true);
130157

131158
assert.deepStrictEqual(replicationInfo, undefined);
132159
});

0 commit comments

Comments
 (0)