Skip to content

Commit f3b04a7

Browse files
added 20kb limit for put bucket policy
Issue : CLDSRV-700
1 parent 1821c3a commit f3b04a7

6 files changed

Lines changed: 301 additions & 4 deletions

File tree

config.json

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,5 +142,9 @@
142142
},
143143
"kmip": {
144144
"providerName": "thales"
145+
},
146+
"apiBodySizeLimits": {
147+
"multiObjectDelete": 2097152,
148+
"bucketPutPolicy": 20480
145149
}
146150
}

constants.js

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -96,11 +96,14 @@ const constants = {
9696
oneMegaBytes: 1024 * 1024,
9797
halfMegaBytes: 512 * 1024,
9898

99-
// Some apis may need a custom body length limit :
100-
apisLengthLimits: {
99+
// Some apis may need a custom body length limit
100+
defaultApiBodySizeLimits: {
101101
// Multi Objects Delete request can be large : up to 1000 keys of 1024 bytes is
102102
// already 1mb, with the other fields it could reach 2mb
103103
'multiObjectDelete': 2 * 1024 * 1024,
104+
// AWS sets the maximum size for bucket policies to 20 KB
105+
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/add-bucket-policy.html
106+
'bucketPutPolicy': 20 * 1024,
104107
},
105108

106109
// hex digest of sha256 hash of empty string:
@@ -266,5 +269,4 @@ const constants = {
266269
onlyOwnerAllowed: ['bucketDeletePolicy', 'bucketGetPolicy', 'bucketPutPolicy'],
267270
};
268271

269-
270272
module.exports = constants;

lib/Config.js

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1730,6 +1730,21 @@ class Config extends EventEmitter {
17301730
}
17311731

17321732
this.supportedLifecycleRules = parseSupportedLifecycleRules(config.supportedLifecycleRules);
1733+
1734+
this.apiBodySizeLimits = { ...constants.defaultApiBodySizeLimits };
1735+
if (config.apiBodySizeLimits) {
1736+
assert(typeof config.apiBodySizeLimits === 'object' &&
1737+
config.apiBodySizeLimits !== null &&
1738+
!Array.isArray(config.apiBodySizeLimits),
1739+
'bad config: apiBodySizeLimits must be an object');
1740+
1741+
for (const [apiKey, limit] of Object.entries(config.apiBodySizeLimits)) {
1742+
assert(Number.isInteger(limit) && limit > 0,
1743+
`bad config: apiBodySizeLimits for "${apiKey}" must be a positive integer`);
1744+
this.apiBodySizeLimits[apiKey] = limit;
1745+
}
1746+
}
1747+
17331748
return config;
17341749
}
17351750

lib/api/api.js

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ const { tagConditionKeyAuth } = require('./apiUtils/authorization/tagConditionKe
7575
const { isRequesterASessionUser } = require('./apiUtils/authorization/permissionChecks');
7676
const checkHttpHeadersSize = require('./apiUtils/object/checkHttpHeadersSize');
7777
const constants = require('../../constants');
78+
const { config } = require('../Config.js');
7879

7980
const monitoringMap = policies.actionMaps.actionMonitoringMapS3;
8081

@@ -223,7 +224,7 @@ const api = {
223224

224225
const defaultMaxPostLength = request.method === 'POST' ?
225226
constants.oneMegaBytes : constants.halfMegaBytes;
226-
const MAX_POST_LENGTH = constants.apisLengthLimits[apiMethod] || defaultMaxPostLength;
227+
const MAX_POST_LENGTH = config.apiBodySizeLimits[apiMethod] || defaultMaxPostLength;
227228
const post = [];
228229
let postLength = 0;
229230
request.on('data', chunk => {

lib/metadata/mpu-bug.js

Lines changed: 244 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,244 @@
1+
#!/usr/bin/env node
2+
3+
// Suppress AWS SDK v2 maintenance mode warnings
4+
process.env.AWS_SDK_JS_SUPPRESS_MAINTENANCE_MODE_MESSAGE = '1';
5+
6+
const AWS = require('aws-sdk');
7+
const crypto = require('crypto');
8+
9+
const CONFIG = {
10+
endpoint: 'http://localhost:8000',
11+
accessKeyId: 'accessKey1',
12+
secretAccessKey: 'verySecretKey1',
13+
region: 'us-east-1',
14+
bucketName: 'mpu-bug',
15+
partSize: 5 * 1024 * 1024,
16+
HTTP: 'http',
17+
};
18+
19+
AWS.config.update({
20+
accessKeyId: CONFIG.accessKeyId,
21+
secretAccessKey: CONFIG.secretAccessKey,
22+
region: CONFIG.region,
23+
sslEnabled: true,
24+
s3ForcePathStyle: true,
25+
httpOptions: {
26+
timeout: 30000,
27+
agent: require('http').Agent({
28+
rejectUnauthorized: false,
29+
keepAlive: true,
30+
maxSockets: 50
31+
})
32+
}
33+
});
34+
const s3 = new AWS.S3({ endpoint: CONFIG.endpoint });
35+
36+
class SimpleMPURaceTester {
37+
constructor() {
38+
this.templateObjectKey = `template-object-${Date.now()}`;
39+
this.templateObjectSize = 15 * 1024 * 1024; // 15MB for 3 parts of 5MB each
40+
}
41+
42+
generateTestData(size = CONFIG.partSize) {
43+
return crypto.randomBytes(size);
44+
}
45+
46+
async checkBucket() {
47+
try {
48+
await s3.headBucket({ Bucket: CONFIG.bucketName }).promise();
49+
console.log(`✅ Bucket accessible: ${CONFIG.bucketName}`);
50+
return true;
51+
} catch (error) {
52+
console.error(`❌ Cannot access bucket: ${error.message}`);
53+
return false;
54+
}
55+
}
56+
57+
async createTemplateObject() {
58+
console.log(`📄 Creating template object (${Math.round(this.templateObjectSize / 1024 / 1024)}MB)...`);
59+
try {
60+
const templateData = this.generateTestData(this.templateObjectSize);
61+
await s3.putObject({
62+
Bucket: CONFIG.bucketName,
63+
Key: this.templateObjectKey,
64+
Body: templateData,
65+
ContentType: 'application/octet-stream'
66+
}).promise();
67+
68+
await s3.headObject({
69+
Bucket: CONFIG.bucketName,
70+
Key: this.templateObjectKey
71+
}).promise();
72+
73+
console.log(`✅ Template object ready: ${this.templateObjectKey}`);
74+
return true;
75+
} catch (error) {
76+
console.error(`❌ Template creation failed: ${error.message}`);
77+
return false;
78+
}
79+
}
80+
81+
async uploadPartCopy(key, uploadId, partNumber, startByte, endByte) {
82+
const params = {
83+
Bucket: CONFIG.bucketName,
84+
Key: key,
85+
UploadId: uploadId,
86+
PartNumber: partNumber,
87+
CopySource: `${CONFIG.bucketName}/${this.templateObjectKey}`,
88+
CopySourceRange: `bytes=${startByte}-${endByte}`
89+
};
90+
91+
try {
92+
const result = await s3.uploadPartCopy(params).promise();
93+
return { ETag: result.ETag, PartNumber: partNumber };
94+
} catch (error) {
95+
console.error(`❌ uploadPartCopy failed for part ${partNumber}: ${error.message}`);
96+
console.error(` Copy source: ${CONFIG.bucketName}/${this.templateObjectKey}`);
97+
console.error(` Range: bytes=${startByte}-${endByte}`);
98+
throw error;
99+
}
100+
}
101+
102+
// Enhanced vulnerability detection
103+
async detectVulnerability(objectKey) {
104+
const versions = await s3.listObjectVersions({
105+
Bucket: CONFIG.bucketName,
106+
Prefix: objectKey
107+
}).promise();
108+
109+
110+
const objectVersions = versions.Versions?.filter(v => v.Key === objectKey) || [];
111+
console.log("VERSIONS ", objectVersions)
112+
console.log(`📊 Check: ${objectVersions.length} versions`);
113+
114+
if (objectVersions.length >= 2) {
115+
console.log(`🎯 MULTIPLE VERSIONS DETECTED! Preserving immediately...`);
116+
objectVersions.forEach((v, i) => {
117+
console.log(` Version ${i+1}: ${v.VersionId} - ETag: ${v.ETag} - Modified: ${v.LastModified}`);
118+
});
119+
}
120+
121+
return objectVersions
122+
}
123+
124+
async createMPU(testId) {
125+
const objectKey = `race-test-${testId}`;
126+
127+
const createResult = await s3.createMultipartUpload({
128+
Bucket: CONFIG.bucketName,
129+
Key: objectKey,
130+
ContentType: 'application/octet-stream'
131+
}).promise();
132+
133+
const uploadId = createResult.UploadId;
134+
135+
const numParts = 3;
136+
const partPromises = [];
137+
138+
for (let partNum = 1; partNum <= numParts; partNum++) {
139+
const startByte = (partNum - 1) * CONFIG.partSize;
140+
const endByte = startByte + CONFIG.partSize - 1;
141+
const copyPromise = this.uploadPartCopy(objectKey, uploadId, partNum, startByte, endByte);
142+
partPromises.push(copyPromise);
143+
}
144+
145+
const parts = await Promise.all(partPromises);
146+
const partsList = parts.map(part => ({
147+
ETag: part.ETag,
148+
PartNumber: part.PartNumber
149+
}));
150+
151+
return { objectKey, uploadId, partsList };
152+
}
153+
154+
async completeMultipartUpload(objectKey, uploadId, partsList) {
155+
try {
156+
console.log(`📦 Completing multipart upload for ${objectKey}...`);
157+
return await s3.completeMultipartUpload({
158+
Bucket: CONFIG.bucketName,
159+
Key: objectKey,
160+
UploadId: uploadId,
161+
MultipartUpload: { Parts: partsList }
162+
}).promise();
163+
} catch (error) {
164+
console.error(`❌ Error during completion: ${error.message}`);
165+
}
166+
}
167+
168+
async createVersionedBucket(bucketName) {
169+
try {
170+
console.log(`📦 Creating versioned bucket: ${bucketName}...`);
171+
await s3.createBucket({ Bucket: bucketName }).promise();
172+
await s3.putBucketVersioning({
173+
Bucket: bucketName,
174+
VersioningConfiguration: {
175+
Status: 'Enabled'
176+
}
177+
}).promise();
178+
console.log(`✅ Versioned bucket created: ${bucketName}`);
179+
} catch (error) {
180+
console.error(`❌ Error creating versioned bucket: ${error}`);
181+
}
182+
}
183+
184+
async deleteObjectByVersionID(objectKey, versionId) {
185+
await s3.deleteObject({
186+
Bucket: CONFIG.bucketName,
187+
Key: objectKey,
188+
VersionId: versionId
189+
}).promise();
190+
}
191+
192+
async getObjectByVersionID(objectKey, versionId) {
193+
try {
194+
const result = await s3.getObject({
195+
Bucket: CONFIG.bucketName,
196+
Key: objectKey,
197+
VersionId: versionId
198+
}).promise();
199+
return result.Body;
200+
} catch (error) {
201+
console.error(`❌ Error getting object by version ID: ${error.message}`);
202+
throw error;
203+
}
204+
}
205+
}
206+
207+
async function main() {
208+
const tester = new SimpleMPURaceTester();
209+
console.log('🔄 Starting testing...');
210+
211+
await tester.createVersionedBucket(CONFIG.bucketName);
212+
213+
if (!(await tester.checkBucket())) {
214+
throw new Error('Cannot access bucket');
215+
}
216+
217+
await tester.createTemplateObject();
218+
const testId = `test-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
219+
220+
const { objectKey, uploadId, partsList } = await tester.createMPU(testId);
221+
console.log('objectkey:', objectKey);
222+
console.log('uploadid:', uploadId);
223+
console.log('partlist:', partsList);
224+
const [result1, result2] = await Promise.all([
225+
tester.completeMultipartUpload(objectKey, uploadId, partsList),
226+
tester.completeMultipartUpload(objectKey, uploadId, partsList),
227+
]);
228+
229+
// await tester.completeMultipartUpload(objectKey, uploadId, partsList)
230+
231+
console.log(`✅ Multipart upload completed: ${result1} and ${result2}`);
232+
console.log("✅ Multipart upload completed:", result1);
233+
console.log("✅ Multipart upload completed:", result2);
234+
235+
// const objectVersions = await tester.detectVulnerability(objectKey);
236+
// await tester.deleteObjectByVersionID(objectKey, objectVersions[0].VersionId)
237+
238+
// // console.log("2ND DETECTION")
239+
// await tester.detectVulnerability(objectKey);
240+
241+
// await tester.getObjectByVersionID(objectKey, objectVersions[1].VersionId)
242+
}
243+
244+
main();

tests/unit/Config.js

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ const {
1313
const {
1414
LOCATION_NAME_DMF,
1515
} = require('../constants');
16+
const constants = require('../../constants');
1617

1718
const { ValidLifecycleRules: supportedLifecycleRules } = require('arsenal').models;
1819

@@ -893,4 +894,34 @@ describe('Config', () => {
893894
assert.strictEqual(config.instanceId.length, 6);
894895
});
895896
});
897+
898+
describe('apisLengthLimits configuration', () => {
899+
let sandbox;
900+
let readFileStub;
901+
902+
beforeEach(() => {
903+
sandbox = sinon.createSandbox();
904+
readFileStub = sandbox.stub(fs, 'readFileSync');
905+
readFileStub.callThrough();
906+
});
907+
908+
afterEach(() => {
909+
sandbox.restore();
910+
});
911+
912+
it('should use default API and overwrite when config is provided', () => {
913+
const multiObjectDeleteSize = 42;
914+
const modifiedConfig = {
915+
...defaultConfig,
916+
apiBodySizeLimits: { 'multiObjectDelete': multiObjectDeleteSize },
917+
};
918+
readFileStub.withArgs(sinon.match(/config.json$/)).returns(JSON.stringify(modifiedConfig));
919+
const config = new ConfigObject();
920+
921+
assert.deepStrictEqual(config.apiBodySizeLimits, {
922+
'multiObjectDelete': multiObjectDeleteSize, // Configured: overwrites default
923+
'bucketPutPolicy': constants.defaultApiBodySizeLimits['bucketPutPolicy'], // Not configured: default
924+
});
925+
});
926+
});
896927
});

0 commit comments

Comments
 (0)