Skip to content

Commit 401021f

Browse files
leif-scalityleif-scality
committed
CLDSRV-883: CreateMultipartUpload parse checksum headers and store them in the MPU overview object
1 parent d396ffe commit 401021f

File tree

6 files changed

+622
-57
lines changed

6 files changed

+622
-57
lines changed

‎lib/api/apiUtils/integrity/validateChecksums.js‎

Lines changed: 91 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ const { Crc32 } = require('@aws-crypto/crc32');
33
const { Crc32c } = require('@aws-crypto/crc32c');
44
const { CrtCrc64Nvme } = require('@aws-sdk/crc64-nvme-crt');
55
const { errors: ArsenalErrors, errorInstances } = require('arsenal');
6+
const { config } = require('../../../Config');
67

78
const errAlgoNotSupported = errorInstances.InvalidRequest.customizeDescription(
89
'The algorithm type you specified in x-amz-checksum- header is invalid.');
@@ -17,7 +18,15 @@ const errTrailerAndChecksum = errorInstances.InvalidRequest.customizeDescription
1718
'Expecting a single x-amz-checksum- header');
1819
const errTrailerNotSupported = errorInstances.InvalidRequest.customizeDescription(
1920
'The value specified in the x-amz-trailer header is not supported');
20-
const { config } = require('../../../Config');
21+
const errMPUAlgoNotSupported = errorInstances.InvalidRequest.customizeDescription(
22+
'Checksum algorithm provided is unsupported. ' +
23+
'Please try again with any of the valid types: ' +
24+
'[CRC32, CRC32C, CRC64NVME, SHA1, SHA256]');
25+
const errMPUTypeInvalid = errorInstances.InvalidRequest.customizeDescription(
26+
'Value for x-amz-checksum-type header is invalid.');
27+
const errMPUTypeWithoutAlgo = errorInstances.InvalidRequest.customizeDescription(
28+
'The x-amz-checksum-type header can only be used ' +
29+
'with the x-amz-checksum-algorithm header.');
2130

2231
const checksumedMethods = Object.freeze({
2332
'completeMultipartUpload': true,
@@ -57,6 +66,10 @@ const ChecksumError = Object.freeze({
5766
TrailerUnexpected: 'TrailerUnexpected',
5867
TrailerAndChecksum: 'TrailerAndChecksum',
5968
TrailerNotSupported: 'TrailerNotSupported',
69+
MPUAlgoNotSupported: 'MPUAlgoNotSupported',
70+
MPUTypeInvalid: 'MPUTypeInvalid',
71+
MPUTypeWithoutAlgo: 'MPUTypeWithoutAlgo',
72+
MPUInvalidCombination: 'MPUInvalidCombination',
6073
});
6174

6275
const base64Regex = /^[A-Za-z0-9+/]*={0,2}$/;
@@ -348,6 +361,16 @@ function arsenalErrorFromChecksumError(err) {
348361
return errTrailerAndChecksum;
349362
case ChecksumError.TrailerNotSupported:
350363
return errTrailerNotSupported;
364+
case ChecksumError.MPUAlgoNotSupported:
365+
return errMPUAlgoNotSupported;
366+
case ChecksumError.MPUTypeInvalid:
367+
return errMPUTypeInvalid;
368+
case ChecksumError.MPUTypeWithoutAlgo:
369+
return errMPUTypeWithoutAlgo;
370+
case ChecksumError.MPUInvalidCombination:
371+
return errorInstances.InvalidRequest.customizeDescription(
372+
`The ${err.details.type} checksum type cannot be used ` +
373+
`with the ${err.details.algorithm} checksum algorithm.`);
351374
default:
352375
return ArsenalErrors.BadDigest;
353376
}
@@ -385,6 +408,72 @@ async function validateMethodChecksumNoChunking(request, body, log) {
385408
return null;
386409
}
387410

411+
const validMPUTypes = new Set(['COMPOSITE', 'FULL_OBJECT']);
412+
const fullObjectAlgorithms = new Set(['crc32', 'crc32c', 'crc64nvme']);
413+
const compositeAlgorithms = new Set(['crc32', 'crc32c', 'sha1', 'sha256']);
414+
415+
const defaultChecksumType = {
416+
crc32: 'COMPOSITE',
417+
crc32c: 'COMPOSITE',
418+
crc64nvme: 'FULL_OBJECT',
419+
sha1: 'COMPOSITE',
420+
sha256: 'COMPOSITE',
421+
};
422+
423+
/**
424+
* Validate x-amz-checksum-algorithm and x-amz-checksum-type headers
425+
* for CreateMultipartUpload.
426+
*
427+
* Validation order mirrors AWS: algorithm first, then type.
428+
*
429+
* @param {object} headers - request headers
430+
* @returns {object} { algorithm, type, isDefault } on success, { error } on failure.
431+
* Defaults to crc64nvme/FULL_OBJECT with isDefault=true when no headers sent.
432+
*/
433+
function getChecksumDataFromMPUHeaders(headers) {
434+
const algorithmHeader = headers['x-amz-checksum-algorithm'];
435+
const typeHeader = headers['x-amz-checksum-type'];
436+
437+
// No checksum headers — use implicit default
438+
if (!algorithmHeader && !typeHeader) {
439+
// isDefault to true means that the checksum won't be returned in listMPUS
440+
return { algorithm: 'crc64nvme', type: defaultChecksumType['crc64nvme'], isDefault: true };
441+
}
442+
443+
// Algorithm first
444+
if (algorithmHeader) {
445+
const algo = algorithmHeader.toLowerCase();
446+
if (!(algo in algorithms)) {
447+
return { error: ChecksumError.MPUAlgoNotSupported, details: { algorithm: algorithmHeader } };
448+
}
449+
}
450+
451+
// Then type
452+
if (typeHeader && !algorithmHeader) {
453+
return { error: ChecksumError.MPUTypeWithoutAlgo, details: { type: typeHeader } };
454+
}
455+
456+
const algo = algorithmHeader.toLowerCase();
457+
458+
if (typeHeader) {
459+
const type = typeHeader.toUpperCase();
460+
if (!validMPUTypes.has(type)) {
461+
return { error: ChecksumError.MPUTypeInvalid, details: { type: typeHeader } };
462+
}
463+
464+
// Validate algorithm + type combination
465+
if ((type === 'FULL_OBJECT' && !fullObjectAlgorithms.has(algo)) ||
466+
(type === 'COMPOSITE' && !compositeAlgorithms.has(algo))) {
467+
return { error: ChecksumError.MPUInvalidCombination, details: { algorithm: algo, type } };
468+
}
469+
470+
return { algorithm: algo, type, isDefault: false };
471+
}
472+
473+
// Only algorithm sent, apply default type
474+
return { algorithm: algo, type: defaultChecksumType[algo], isDefault: false };
475+
}
476+
388477
module.exports = {
389478
ChecksumError,
390479
validateChecksumsNoChunking,
@@ -393,4 +482,5 @@ module.exports = {
393482
arsenalErrorFromChecksumError,
394483
algorithms,
395484
checksumedMethods,
485+
getChecksumDataFromMPUHeaders,
396486
};

‎lib/api/initiateMultipartUpload.js‎

Lines changed: 76 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ const { getObjectSSEConfiguration } = require('./apiUtils/bucket/bucketEncryptio
2424
const { setExpirationHeaders } = require('./apiUtils/object/expirationHeaders');
2525
const { setSSEHeaders } = require('./apiUtils/object/sseHeaders');
2626
const { updateEncryption } = require('./apiUtils/bucket/updateEncryption');
27+
const { getChecksumDataFromMPUHeaders, arsenalErrorFromChecksumError } =
28+
require('./apiUtils/integrity/validateChecksums');
2729
const { config } = require('../Config');
2830
const kms = require('../kms/wrapper');
2931

@@ -87,6 +89,13 @@ function initiateMultipartUpload(authInfo, request, log, callback) {
8789
`value ${websiteRedirectHeader}`, { error: err });
8890
return callback(err);
8991
}
92+
const checksumConfig = getChecksumDataFromMPUHeaders(request.headers);
93+
if (checksumConfig.error) {
94+
const checksumErr = arsenalErrorFromChecksumError(checksumConfig);
95+
log.debug('checksum header validation failed', { error: checksumErr, method: 'initiateMultipartUpload' });
96+
monitoring.promMetrics('PUT', bucketName, checksumErr.code, 'initiateMultipartUpload');
97+
return callback(checksumErr);
98+
}
9099
const metaHeaders = getMetaHeaders(request.headers);
91100
if (metaHeaders instanceof Error) {
92101
log.debug('user metadata validation failed', {
@@ -145,13 +154,15 @@ function initiateMultipartUpload(authInfo, request, log, callback) {
145154
initiatorDisplayName,
146155
splitter: constants.splitter,
147156
};
157+
metadataStoreParams.checksumAlgorithm = checksumConfig.algorithm;
158+
metadataStoreParams.checksumType = checksumConfig.type;
159+
metadataStoreParams.checksumIsDefault = checksumConfig.isDefault;
148160
const tagging = request.headers['x-amz-tagging'];
149161
if (tagging) {
150162
metadataStoreParams.tagging = tagging;
151163
}
152164

153-
function _getMPUBucket(destinationBucket, log, corsHeaders,
154-
uploadId, cipherBundle, locConstraint, callback) {
165+
function _getMPUBucket(destinationBucket, log, corsHeaders, uploadId, cipherBundle, locConstraint, callback) {
155166
const xmlParams = {
156167
bucketName,
157168
objectKey,
@@ -205,6 +216,14 @@ function initiateMultipartUpload(authInfo, request, log, callback) {
205216
mpuMD['x-amz-server-side-encryption'],
206217
mpuMD['x-amz-server-side-encryption-aws-kms-key-id']);
207218

219+
// Only respond the headers if the user sent them
220+
if (!checksumConfig.isDefault) {
221+
// eslint-disable-next-line no-param-reassign
222+
corsHeaders['x-amz-checksum-algorithm'] = checksumConfig.algorithm.toUpperCase();
223+
// eslint-disable-next-line no-param-reassign
224+
corsHeaders['x-amz-checksum-type'] = checksumConfig.type;
225+
}
226+
208227
monitoring.promMetrics('PUT', bucketName, '200',
209228
'initiateMultipartUpload');
210229
return callback(null, xml, corsHeaders);
@@ -279,47 +298,48 @@ function initiateMultipartUpload(authInfo, request, log, callback) {
279298
}
280299

281300
return data.initiateMPU(mpuInfo, websiteRedirectHeader, log,
282-
(err, dataBackendResObj, isVersionedObj) => {
283-
// will return as true and a custom error if external backend does
284-
// not support versioned objects
285-
if (isVersionedObj) {
286-
monitoring.promMetrics('PUT', bucketName, 501,
287-
'initiateMultipartUpload');
288-
return callback(err);
289-
}
290-
if (err) {
291-
monitoring.promMetrics('PUT', bucketName, err.code,
292-
'initiateMultipartUpload');
293-
return callback(err);
294-
}
295-
// if mpu not handled externally, dataBackendResObj will be null
296-
if (dataBackendResObj) {
297-
uploadId = dataBackendResObj.UploadId;
298-
} else {
299-
// Generate uniqueID without dashes so routing not messed up
300-
uploadId = uuidv4().replace(/-/g, '');
301-
}
302-
return _getMPUBucket(destinationBucket, log, corsHeaders,
303-
uploadId, cipherBundle, locConstraint, callback);
304-
});
301+
(err, dataBackendResObj, isVersionedObj) => {
302+
// will return as true and a custom error if external backend does
303+
// not support versioned objects
304+
if (isVersionedObj) {
305+
monitoring.promMetrics('PUT', bucketName, 501,
306+
'initiateMultipartUpload');
307+
return callback(err);
308+
}
309+
if (err) {
310+
monitoring.promMetrics('PUT', bucketName, err.code,
311+
'initiateMultipartUpload');
312+
return callback(err);
313+
}
314+
// if mpu not handled externally, dataBackendResObj will be null
315+
if (dataBackendResObj) {
316+
uploadId = dataBackendResObj.UploadId;
317+
} else {
318+
// Generate uniqueID without dashes so routing not messed up
319+
uploadId = uuidv4().replace(/-/g, '');
320+
}
321+
return _getMPUBucket(destinationBucket, log, corsHeaders,
322+
uploadId, cipherBundle, locConstraint, callback);
323+
});
305324
}
306325

307326
async.waterfall([
308327
next => standardMetadataValidateBucketAndObj(metadataValParams, request.actionImplicitDenies, log,
309328
(error, destinationBucket, destObjMD) =>
310329
updateEncryption(error, destinationBucket, destObjMD, objectKey, log, { skipObject: true },
311-
(error, destinationBucket) => {
312-
const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, destinationBucket);
313-
if (error) {
314-
log.debug('error processing request', {
315-
error,
316-
method: 'metadataValidateBucketAndObj',
317-
});
318-
monitoring.promMetrics('PUT', bucketName, error.code, 'initiateMultipartUpload');
319-
return next(error, corsHeaders);
320-
}
321-
return next(null, corsHeaders, destinationBucket);
322-
})),
330+
(error, destinationBucket) => {
331+
const corsHeaders = collectCorsHeaders(
332+
request.headers.origin, request.method, destinationBucket);
333+
if (error) {
334+
log.debug('error processing request', {
335+
error,
336+
method: 'metadataValidateBucketAndObj',
337+
});
338+
monitoring.promMetrics('PUT', bucketName, error.code, 'initiateMultipartUpload');
339+
return next(error, corsHeaders);
340+
}
341+
return next(null, corsHeaders, destinationBucket);
342+
})),
323343
(corsHeaders, destinationBucket, next) => {
324344
if (destinationBucket.hasDeletedFlag() && accountCanonicalID !== destinationBucket.getOwner()) {
325345
log.trace('deleted flag on bucket and request from non-owner account');
@@ -329,25 +349,25 @@ function initiateMultipartUpload(authInfo, request, log, callback) {
329349
if (destinationBucket.hasTransientFlag() || destinationBucket.hasDeletedFlag()) {
330350
log.trace('transient or deleted flag so cleaning up bucket');
331351
return cleanUpBucket(
332-
destinationBucket,
333-
accountCanonicalID,
334-
log,
335-
error => {
336-
if (error) {
337-
log.debug('error cleaning up bucket with flag',
338-
{
339-
error,
340-
transientFlag: destinationBucket.hasTransientFlag(),
341-
deletedFlag: destinationBucket.hasDeletedFlag(),
342-
});
343-
// To avoid confusing user with error
344-
// from cleaning up
345-
// bucket return InternalError
346-
monitoring.promMetrics('PUT', bucketName, 500, 'initiateMultipartUpload');
347-
return next(errors.InternalError, corsHeaders);
348-
}
349-
return next(null, corsHeaders, destinationBucket);
350-
});
352+
destinationBucket,
353+
accountCanonicalID,
354+
log,
355+
error => {
356+
if (error) {
357+
log.debug('error cleaning up bucket with flag',
358+
{
359+
error,
360+
transientFlag: destinationBucket.hasTransientFlag(),
361+
deletedFlag: destinationBucket.hasDeletedFlag(),
362+
});
363+
// To avoid confusing user with error
364+
// from cleaning up
365+
// bucket return InternalError
366+
monitoring.promMetrics('PUT', bucketName, 500, 'initiateMultipartUpload');
367+
return next(errors.InternalError, corsHeaders);
368+
}
369+
return next(null, corsHeaders, destinationBucket);
370+
});
351371
}
352372
return next(null, corsHeaders, destinationBucket);
353373
},

‎lib/services.js‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -587,6 +587,9 @@ const services = {
587587
if (params.legalHold) {
588588
multipartObjectMD.legalHold = params.legalHold;
589589
}
590+
multipartObjectMD.checksumAlgorithm = params.checksumAlgorithm;
591+
multipartObjectMD.checksumType = params.checksumType;
592+
multipartObjectMD.checksumIsDefault = params.checksumIsDefault;
590593
Object.keys(params.metaHeaders).forEach(val => {
591594
multipartObjectMD[val] = params.metaHeaders[val];
592595
});

0 commit comments

Comments
 (0)