Skip to content

CLDSRV-883: CreateMultipartUpload parse checksum headers and store them in MPU overview object #6144

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
leif-scality wants to merge 1 commit into
base: development/9.4
Choose a base branch
from
Open

CLDSRV-883: CreateMultipartUpload parse checksum headers and store them in MPU overview object #6144

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 91 additions & 1 deletion lib/api/apiUtils/integrity/validateChecksums.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ const { Crc32 } = require('@aws-crypto/crc32');
const { Crc32c } = require('@aws-crypto/crc32c');
const { CrtCrc64Nvme } = require('@aws-sdk/crc64-nvme-crt');
const { errors: ArsenalErrors, errorInstances } = require('arsenal');
const { config } = require('../../../Config');

const errAlgoNotSupported = errorInstances.InvalidRequest.customizeDescription(
'The algorithm type you specified in x-amz-checksum- header is invalid.');
Expand All @@ -17,7 +18,15 @@ const errTrailerAndChecksum = errorInstances.InvalidRequest.customizeDescription
'Expecting a single x-amz-checksum- header');
const errTrailerNotSupported = errorInstances.InvalidRequest.customizeDescription(
'The value specified in the x-amz-trailer header is not supported');
const { config } = require('../../../Config');
const errMPUAlgoNotSupported = errorInstances.InvalidRequest.customizeDescription(
'Checksum algorithm provided is unsupported. ' +
'Please try again with any of the valid types: ' +
'[CRC32, CRC32C, CRC64NVME, SHA1, SHA256]');
const errMPUTypeInvalid = errorInstances.InvalidRequest.customizeDescription(
'Value for x-amz-checksum-type header is invalid.');
const errMPUTypeWithoutAlgo = errorInstances.InvalidRequest.customizeDescription(
'The x-amz-checksum-type header can only be used ' +
'with the x-amz-checksum-algorithm header.');

const checksumedMethods = Object.freeze({
'completeMultipartUpload': true,
Expand Down Expand Up @@ -57,6 +66,10 @@ const ChecksumError = Object.freeze({
TrailerUnexpected: 'TrailerUnexpected',
TrailerAndChecksum: 'TrailerAndChecksum',
TrailerNotSupported: 'TrailerNotSupported',
MPUAlgoNotSupported: 'MPUAlgoNotSupported',
MPUTypeInvalid: 'MPUTypeInvalid',
MPUTypeWithoutAlgo: 'MPUTypeWithoutAlgo',
MPUInvalidCombination: 'MPUInvalidCombination',
});

const base64Regex = /^[A-Za-z0-9+/]*={0,2}$/;
Expand Down Expand Up @@ -348,6 +361,16 @@ function arsenalErrorFromChecksumError(err) {
return errTrailerAndChecksum;
case ChecksumError.TrailerNotSupported:
return errTrailerNotSupported;
case ChecksumError.MPUAlgoNotSupported:
return errMPUAlgoNotSupported;
case ChecksumError.MPUTypeInvalid:
return errMPUTypeInvalid;
case ChecksumError.MPUTypeWithoutAlgo:
return errMPUTypeWithoutAlgo;
case ChecksumError.MPUInvalidCombination:
return errorInstances.InvalidRequest.customizeDescription(
`The ${err.details.type} checksum type cannot be used ` +
`with the ${err.details.algorithm} checksum algorithm.`);
default:
return ArsenalErrors.BadDigest;
}
Expand Down Expand Up @@ -385,6 +408,72 @@ async function validateMethodChecksumNoChunking(request, body, log) {
return null;
}

const validMPUTypes = new Set(['COMPOSITE', 'FULL_OBJECT']);
const fullObjectAlgorithms = new Set(['crc32', 'crc32c', 'crc64nvme']);
const compositeAlgorithms = new Set(['crc32', 'crc32c', 'sha1', 'sha256']);

const defaultChecksumType = {
crc32: 'COMPOSITE',
crc32c: 'COMPOSITE',
crc64nvme: 'FULL_OBJECT',
sha1: 'COMPOSITE',
sha256: 'COMPOSITE',
};

/**
* Validate x-amz-checksum-algorithm and x-amz-checksum-type headers
* for CreateMultipartUpload.
*
* Validation order mirrors AWS: algorithm first, then type.
*
* @param {object} headers - request headers
* @returns {object} { algorithm, type, isDefault } on success, { error } on failure.
* Defaults to crc64nvme/FULL_OBJECT with isDefault=true when no headers sent.
*/
function getChecksumDataFromMPUHeaders(headers) {
const algorithmHeader = headers['x-amz-checksum-algorithm'];
const typeHeader = headers['x-amz-checksum-type'];

// No checksum headers — use implicit default
if (!algorithmHeader && !typeHeader) {
// isDefault to true means that the checksum won't be returned in listMPUS
return { algorithm: 'crc64nvme', type: defaultChecksumType['crc64nvme'], isDefault: true };
}

// Algorithm first
if (algorithmHeader) {
const algo = algorithmHeader.toLowerCase();
if (!(algo in algorithms)) {
return { error: ChecksumError.MPUAlgoNotSupported, details: { algorithm: algorithmHeader } };
}
}

// Then type
if (typeHeader && !algorithmHeader) {
return { error: ChecksumError.MPUTypeWithoutAlgo, details: { type: typeHeader } };
}

const algo = algorithmHeader.toLowerCase();

if (typeHeader) {
const type = typeHeader.toUpperCase();
if (!validMPUTypes.has(type)) {
return { error: ChecksumError.MPUTypeInvalid, details: { type: typeHeader } };
}

// Validate algorithm + type combination
if ((type === 'FULL_OBJECT' && !fullObjectAlgorithms.has(algo)) ||
(type === 'COMPOSITE' && !compositeAlgorithms.has(algo))) {
return { error: ChecksumError.MPUInvalidCombination, details: { algorithm: algo, type } };
}

return { algorithm: algo, type, isDefault: false };
}

// Only algorithm sent, apply default type
return { algorithm: algo, type: defaultChecksumType[algo], isDefault: false };
}

module.exports = {
ChecksumError,
validateChecksumsNoChunking,
Expand All @@ -393,4 +482,5 @@ module.exports = {
arsenalErrorFromChecksumError,
algorithms,
checksumedMethods,
getChecksumDataFromMPUHeaders,
};
132 changes: 76 additions & 56 deletions lib/api/initiateMultipartUpload.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ const { getObjectSSEConfiguration } = require('./apiUtils/bucket/bucketEncryptio
const { setExpirationHeaders } = require('./apiUtils/object/expirationHeaders');
const { setSSEHeaders } = require('./apiUtils/object/sseHeaders');
const { updateEncryption } = require('./apiUtils/bucket/updateEncryption');
const { getChecksumDataFromMPUHeaders, arsenalErrorFromChecksumError } =
require('./apiUtils/integrity/validateChecksums');
const { config } = require('../Config');
const kms = require('../kms/wrapper');

Expand Down Expand Up @@ -87,6 +89,13 @@ function initiateMultipartUpload(authInfo, request, log, callback) {
`value ${websiteRedirectHeader}`, { error: err });
return callback(err);
}
const checksumConfig = getChecksumDataFromMPUHeaders(request.headers);
if (checksumConfig.error) {
const checksumErr = arsenalErrorFromChecksumError(checksumConfig);
log.debug('checksum header validation failed', { error: checksumErr, method: 'initiateMultipartUpload' });
monitoring.promMetrics('PUT', bucketName, checksumErr.code, 'initiateMultipartUpload');
return callback(checksumErr);
}
const metaHeaders = getMetaHeaders(request.headers);
if (metaHeaders instanceof Error) {
log.debug('user metadata validation failed', {
Expand Down Expand Up @@ -145,13 +154,15 @@ function initiateMultipartUpload(authInfo, request, log, callback) {
initiatorDisplayName,
splitter: constants.splitter,
};
metadataStoreParams.checksumAlgorithm = checksumConfig.algorithm;
metadataStoreParams.checksumType = checksumConfig.type;
metadataStoreParams.checksumIsDefault = checksumConfig.isDefault;
const tagging = request.headers['x-amz-tagging'];
if (tagging) {
metadataStoreParams.tagging = tagging;
}

function _getMPUBucket(destinationBucket, log, corsHeaders,
uploadId, cipherBundle, locConstraint, callback) {
function _getMPUBucket(destinationBucket, log, corsHeaders, uploadId, cipherBundle, locConstraint, callback) {
const xmlParams = {
bucketName,
objectKey,
Expand Down Expand Up @@ -205,6 +216,14 @@ function initiateMultipartUpload(authInfo, request, log, callback) {
mpuMD['x-amz-server-side-encryption'],
mpuMD['x-amz-server-side-encryption-aws-kms-key-id']);

// Only respond the headers if the user sent them
if (!checksumConfig.isDefault) {
// eslint-disable-next-line no-param-reassign
corsHeaders['x-amz-checksum-algorithm'] = checksumConfig.algorithm.toUpperCase();
// eslint-disable-next-line no-param-reassign
corsHeaders['x-amz-checksum-type'] = checksumConfig.type;
}

monitoring.promMetrics('PUT', bucketName, '200',
'initiateMultipartUpload');
return callback(null, xml, corsHeaders);
Expand Down Expand Up @@ -279,47 +298,48 @@ function initiateMultipartUpload(authInfo, request, log, callback) {
}

return data.initiateMPU(mpuInfo, websiteRedirectHeader, log,
(err, dataBackendResObj, isVersionedObj) => {
// will return as true and a custom error if external backend does
// not support versioned objects
if (isVersionedObj) {
monitoring.promMetrics('PUT', bucketName, 501,
'initiateMultipartUpload');
return callback(err);
}
if (err) {
monitoring.promMetrics('PUT', bucketName, err.code,
'initiateMultipartUpload');
return callback(err);
}
// if mpu not handled externally, dataBackendResObj will be null
if (dataBackendResObj) {
uploadId = dataBackendResObj.UploadId;
} else {
// Generate uniqueID without dashes so routing not messed up
uploadId = uuidv4().replace(/-/g, '');
}
return _getMPUBucket(destinationBucket, log, corsHeaders,
uploadId, cipherBundle, locConstraint, callback);
});
(err, dataBackendResObj, isVersionedObj) => {
// will return as true and a custom error if external backend does
// not support versioned objects
if (isVersionedObj) {
monitoring.promMetrics('PUT', bucketName, 501,
'initiateMultipartUpload');
return callback(err);
}
if (err) {
monitoring.promMetrics('PUT', bucketName, err.code,
'initiateMultipartUpload');
return callback(err);
}
// if mpu not handled externally, dataBackendResObj will be null
if (dataBackendResObj) {
uploadId = dataBackendResObj.UploadId;
} else {
// Generate uniqueID without dashes so routing not messed up
uploadId = uuidv4().replace(/-/g, '');
}
return _getMPUBucket(destinationBucket, log, corsHeaders,
uploadId, cipherBundle, locConstraint, callback);
});
}

async.waterfall([
next => standardMetadataValidateBucketAndObj(metadataValParams, request.actionImplicitDenies, log,
(error, destinationBucket, destObjMD) =>
updateEncryption(error, destinationBucket, destObjMD, objectKey, log, { skipObject: true },
(error, destinationBucket) => {
const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, destinationBucket);
if (error) {
log.debug('error processing request', {
error,
method: 'metadataValidateBucketAndObj',
});
monitoring.promMetrics('PUT', bucketName, error.code, 'initiateMultipartUpload');
return next(error, corsHeaders);
}
return next(null, corsHeaders, destinationBucket);
})),
(error, destinationBucket) => {
const corsHeaders = collectCorsHeaders(
request.headers.origin, request.method, destinationBucket);
if (error) {
log.debug('error processing request', {
error,
method: 'metadataValidateBucketAndObj',
});
monitoring.promMetrics('PUT', bucketName, error.code, 'initiateMultipartUpload');
return next(error, corsHeaders);
}
return next(null, corsHeaders, destinationBucket);
})),
(corsHeaders, destinationBucket, next) => {
if (destinationBucket.hasDeletedFlag() && accountCanonicalID !== destinationBucket.getOwner()) {
log.trace('deleted flag on bucket and request from non-owner account');
Expand All @@ -329,25 +349,25 @@ function initiateMultipartUpload(authInfo, request, log, callback) {
if (destinationBucket.hasTransientFlag() || destinationBucket.hasDeletedFlag()) {
log.trace('transient or deleted flag so cleaning up bucket');
return cleanUpBucket(
destinationBucket,
accountCanonicalID,
log,
error => {
if (error) {
log.debug('error cleaning up bucket with flag',
{
error,
transientFlag: destinationBucket.hasTransientFlag(),
deletedFlag: destinationBucket.hasDeletedFlag(),
});
// To avoid confusing user with error
// from cleaning up
// bucket return InternalError
monitoring.promMetrics('PUT', bucketName, 500, 'initiateMultipartUpload');
return next(errors.InternalError, corsHeaders);
}
return next(null, corsHeaders, destinationBucket);
});
destinationBucket,
accountCanonicalID,
log,
error => {
if (error) {
log.debug('error cleaning up bucket with flag',
{
error,
transientFlag: destinationBucket.hasTransientFlag(),
deletedFlag: destinationBucket.hasDeletedFlag(),
});
// To avoid confusing user with error
// from cleaning up
// bucket return InternalError
monitoring.promMetrics('PUT', bucketName, 500, 'initiateMultipartUpload');
return next(errors.InternalError, corsHeaders);
}
return next(null, corsHeaders, destinationBucket);
});
}
return next(null, corsHeaders, destinationBucket);
},
Expand Down
3 changes: 3 additions & 0 deletions lib/services.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -587,6 +587,9 @@ const services = {
if (params.legalHold) {
multipartObjectMD.legalHold = params.legalHold;
}
multipartObjectMD.checksumAlgorithm = params.checksumAlgorithm;
multipartObjectMD.checksumType = params.checksumType;
multipartObjectMD.checksumIsDefault = params.checksumIsDefault;
Object.keys(params.metaHeaders).forEach(val => {
multipartObjectMD[val] = params.metaHeaders[val];
});
Expand Down
Loading
Loading