diff --git a/lib/api/apiUtils/integrity/validateChecksums.js b/lib/api/apiUtils/integrity/validateChecksums.js index fe79228394..b2e08fa8f9 100644 --- a/lib/api/apiUtils/integrity/validateChecksums.js +++ b/lib/api/apiUtils/integrity/validateChecksums.js @@ -3,6 +3,7 @@ const { Crc32 } = require('@aws-crypto/crc32'); const { Crc32c } = require('@aws-crypto/crc32c'); const { CrtCrc64Nvme } = require('@aws-sdk/crc64-nvme-crt'); const { errors: ArsenalErrors, errorInstances } = require('arsenal'); +const { config } = require('../../../Config'); const errAlgoNotSupported = errorInstances.InvalidRequest.customizeDescription( 'The algorithm type you specified in x-amz-checksum- header is invalid.'); @@ -17,7 +18,15 @@ const errTrailerAndChecksum = errorInstances.InvalidRequest.customizeDescription 'Expecting a single x-amz-checksum- header'); const errTrailerNotSupported = errorInstances.InvalidRequest.customizeDescription( 'The value specified in the x-amz-trailer header is not supported'); -const { config } = require('../../../Config'); +const errMPUAlgoNotSupported = errorInstances.InvalidRequest.customizeDescription( + 'Checksum algorithm provided is unsupported. ' + + 'Please try again with any of the valid types: ' + + '[CRC32, CRC32C, CRC64NVME, SHA1, SHA256]'); +const errMPUTypeInvalid = errorInstances.InvalidRequest.customizeDescription( + 'Value for x-amz-checksum-type header is invalid.'); +const errMPUTypeWithoutAlgo = errorInstances.InvalidRequest.customizeDescription( + 'The x-amz-checksum-type header can only be used ' + + 'with the x-amz-checksum-algorithm header.'); const checksumedMethods = Object.freeze({ 'completeMultipartUpload': true, @@ -57,6 +66,10 @@ const ChecksumError = Object.freeze({ TrailerUnexpected: 'TrailerUnexpected', TrailerAndChecksum: 'TrailerAndChecksum', TrailerNotSupported: 'TrailerNotSupported', + MPUAlgoNotSupported: 'MPUAlgoNotSupported', + MPUTypeInvalid: 'MPUTypeInvalid', + MPUTypeWithoutAlgo: 'MPUTypeWithoutAlgo', + MPUInvalidCombination: 'MPUInvalidCombination', }); const base64Regex = /^[A-Za-z0-9+/]*={0,2}$/; @@ -348,6 +361,16 @@ function arsenalErrorFromChecksumError(err) { return errTrailerAndChecksum; case ChecksumError.TrailerNotSupported: return errTrailerNotSupported; + case ChecksumError.MPUAlgoNotSupported: + return errMPUAlgoNotSupported; + case ChecksumError.MPUTypeInvalid: + return errMPUTypeInvalid; + case ChecksumError.MPUTypeWithoutAlgo: + return errMPUTypeWithoutAlgo; + case ChecksumError.MPUInvalidCombination: + return errorInstances.InvalidRequest.customizeDescription( + `The ${err.details.type} checksum type cannot be used ` + + `with the ${err.details.algorithm} checksum algorithm.`); default: return ArsenalErrors.BadDigest; } @@ -385,6 +408,72 @@ async function validateMethodChecksumNoChunking(request, body, log) { return null; } +const validMPUTypes = new Set(['COMPOSITE', 'FULL_OBJECT']); +const fullObjectAlgorithms = new Set(['crc32', 'crc32c', 'crc64nvme']); +const compositeAlgorithms = new Set(['crc32', 'crc32c', 'sha1', 'sha256']); + +const defaultChecksumType = { + crc32: 'COMPOSITE', + crc32c: 'COMPOSITE', + crc64nvme: 'FULL_OBJECT', + sha1: 'COMPOSITE', + sha256: 'COMPOSITE', +}; + +/** + * Validate x-amz-checksum-algorithm and x-amz-checksum-type headers + * for CreateMultipartUpload. + * + * Validation order mirrors AWS: algorithm first, then type. + * + * @param {object} headers - request headers + * @returns {object} { algorithm, type, isDefault } on success, { error } on failure. + * Defaults to crc64nvme/FULL_OBJECT with isDefault=true when no headers sent. + */ +function getChecksumDataFromMPUHeaders(headers) { + const algorithmHeader = headers['x-amz-checksum-algorithm']; + const typeHeader = headers['x-amz-checksum-type']; + + // No checksum headers — use implicit default + if (!algorithmHeader && !typeHeader) { + // isDefault to true means that the checksum won't be returned in listMPUS + return { algorithm: 'crc64nvme', type: defaultChecksumType['crc64nvme'], isDefault: true }; + } + + // Algorithm first + if (algorithmHeader) { + const algo = algorithmHeader.toLowerCase(); + if (!(algo in algorithms)) { + return { error: ChecksumError.MPUAlgoNotSupported, details: { algorithm: algorithmHeader } }; + } + } + + // Then type + if (typeHeader && !algorithmHeader) { + return { error: ChecksumError.MPUTypeWithoutAlgo, details: { type: typeHeader } }; + } + + const algo = algorithmHeader.toLowerCase(); + + if (typeHeader) { + const type = typeHeader.toUpperCase(); + if (!validMPUTypes.has(type)) { + return { error: ChecksumError.MPUTypeInvalid, details: { type: typeHeader } }; + } + + // Validate algorithm + type combination + if ((type === 'FULL_OBJECT' && !fullObjectAlgorithms.has(algo)) || + (type === 'COMPOSITE' && !compositeAlgorithms.has(algo))) { + return { error: ChecksumError.MPUInvalidCombination, details: { algorithm: algo, type } }; + } + + return { algorithm: algo, type, isDefault: false }; + } + + // Only algorithm sent, apply default type + return { algorithm: algo, type: defaultChecksumType[algo], isDefault: false }; +} + module.exports = { ChecksumError, validateChecksumsNoChunking, @@ -393,4 +482,5 @@ module.exports = { arsenalErrorFromChecksumError, algorithms, checksumedMethods, + getChecksumDataFromMPUHeaders, }; diff --git a/lib/api/initiateMultipartUpload.js b/lib/api/initiateMultipartUpload.js index 3cc8e17f02..bc6aa0df29 100644 --- a/lib/api/initiateMultipartUpload.js +++ b/lib/api/initiateMultipartUpload.js @@ -24,6 +24,8 @@ const { getObjectSSEConfiguration } = require('./apiUtils/bucket/bucketEncryptio const { setExpirationHeaders } = require('./apiUtils/object/expirationHeaders'); const { setSSEHeaders } = require('./apiUtils/object/sseHeaders'); const { updateEncryption } = require('./apiUtils/bucket/updateEncryption'); +const { getChecksumDataFromMPUHeaders, arsenalErrorFromChecksumError } = + require('./apiUtils/integrity/validateChecksums'); const { config } = require('../Config'); const kms = require('../kms/wrapper'); @@ -87,6 +89,13 @@ function initiateMultipartUpload(authInfo, request, log, callback) { `value ${websiteRedirectHeader}`, { error: err }); return callback(err); } + const checksumConfig = getChecksumDataFromMPUHeaders(request.headers); + if (checksumConfig.error) { + const checksumErr = arsenalErrorFromChecksumError(checksumConfig); + log.debug('checksum header validation failed', { error: checksumErr, method: 'initiateMultipartUpload' }); + monitoring.promMetrics('PUT', bucketName, checksumErr.code, 'initiateMultipartUpload'); + return callback(checksumErr); + } const metaHeaders = getMetaHeaders(request.headers); if (metaHeaders instanceof Error) { log.debug('user metadata validation failed', { @@ -145,13 +154,15 @@ function initiateMultipartUpload(authInfo, request, log, callback) { initiatorDisplayName, splitter: constants.splitter, }; + metadataStoreParams.checksumAlgorithm = checksumConfig.algorithm; + metadataStoreParams.checksumType = checksumConfig.type; + metadataStoreParams.checksumIsDefault = checksumConfig.isDefault; const tagging = request.headers['x-amz-tagging']; if (tagging) { metadataStoreParams.tagging = tagging; } - function _getMPUBucket(destinationBucket, log, corsHeaders, - uploadId, cipherBundle, locConstraint, callback) { + function _getMPUBucket(destinationBucket, log, corsHeaders, uploadId, cipherBundle, locConstraint, callback) { const xmlParams = { bucketName, objectKey, @@ -205,6 +216,14 @@ function initiateMultipartUpload(authInfo, request, log, callback) { mpuMD['x-amz-server-side-encryption'], mpuMD['x-amz-server-side-encryption-aws-kms-key-id']); + // Only respond the headers if the user sent them + if (!checksumConfig.isDefault) { + // eslint-disable-next-line no-param-reassign + corsHeaders['x-amz-checksum-algorithm'] = checksumConfig.algorithm.toUpperCase(); + // eslint-disable-next-line no-param-reassign + corsHeaders['x-amz-checksum-type'] = checksumConfig.type; + } + monitoring.promMetrics('PUT', bucketName, '200', 'initiateMultipartUpload'); return callback(null, xml, corsHeaders); @@ -279,47 +298,48 @@ function initiateMultipartUpload(authInfo, request, log, callback) { } return data.initiateMPU(mpuInfo, websiteRedirectHeader, log, - (err, dataBackendResObj, isVersionedObj) => { - // will return as true and a custom error if external backend does - // not support versioned objects - if (isVersionedObj) { - monitoring.promMetrics('PUT', bucketName, 501, - 'initiateMultipartUpload'); - return callback(err); - } - if (err) { - monitoring.promMetrics('PUT', bucketName, err.code, - 'initiateMultipartUpload'); - return callback(err); - } - // if mpu not handled externally, dataBackendResObj will be null - if (dataBackendResObj) { - uploadId = dataBackendResObj.UploadId; - } else { - // Generate uniqueID without dashes so routing not messed up - uploadId = uuidv4().replace(/-/g, ''); - } - return _getMPUBucket(destinationBucket, log, corsHeaders, - uploadId, cipherBundle, locConstraint, callback); - }); + (err, dataBackendResObj, isVersionedObj) => { + // will return as true and a custom error if external backend does + // not support versioned objects + if (isVersionedObj) { + monitoring.promMetrics('PUT', bucketName, 501, + 'initiateMultipartUpload'); + return callback(err); + } + if (err) { + monitoring.promMetrics('PUT', bucketName, err.code, + 'initiateMultipartUpload'); + return callback(err); + } + // if mpu not handled externally, dataBackendResObj will be null + if (dataBackendResObj) { + uploadId = dataBackendResObj.UploadId; + } else { + // Generate uniqueID without dashes so routing not messed up + uploadId = uuidv4().replace(/-/g, ''); + } + return _getMPUBucket(destinationBucket, log, corsHeaders, + uploadId, cipherBundle, locConstraint, callback); + }); } async.waterfall([ next => standardMetadataValidateBucketAndObj(metadataValParams, request.actionImplicitDenies, log, (error, destinationBucket, destObjMD) => updateEncryption(error, destinationBucket, destObjMD, objectKey, log, { skipObject: true }, - (error, destinationBucket) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, destinationBucket); - if (error) { - log.debug('error processing request', { - error, - method: 'metadataValidateBucketAndObj', - }); - monitoring.promMetrics('PUT', bucketName, error.code, 'initiateMultipartUpload'); - return next(error, corsHeaders); - } - return next(null, corsHeaders, destinationBucket); - })), + (error, destinationBucket) => { + const corsHeaders = collectCorsHeaders( + request.headers.origin, request.method, destinationBucket); + if (error) { + log.debug('error processing request', { + error, + method: 'metadataValidateBucketAndObj', + }); + monitoring.promMetrics('PUT', bucketName, error.code, 'initiateMultipartUpload'); + return next(error, corsHeaders); + } + return next(null, corsHeaders, destinationBucket); + })), (corsHeaders, destinationBucket, next) => { if (destinationBucket.hasDeletedFlag() && accountCanonicalID !== destinationBucket.getOwner()) { log.trace('deleted flag on bucket and request from non-owner account'); @@ -329,25 +349,25 @@ function initiateMultipartUpload(authInfo, request, log, callback) { if (destinationBucket.hasTransientFlag() || destinationBucket.hasDeletedFlag()) { log.trace('transient or deleted flag so cleaning up bucket'); return cleanUpBucket( - destinationBucket, - accountCanonicalID, - log, - error => { - if (error) { - log.debug('error cleaning up bucket with flag', - { - error, - transientFlag: destinationBucket.hasTransientFlag(), - deletedFlag: destinationBucket.hasDeletedFlag(), - }); - // To avoid confusing user with error - // from cleaning up - // bucket return InternalError - monitoring.promMetrics('PUT', bucketName, 500, 'initiateMultipartUpload'); - return next(errors.InternalError, corsHeaders); - } - return next(null, corsHeaders, destinationBucket); - }); + destinationBucket, + accountCanonicalID, + log, + error => { + if (error) { + log.debug('error cleaning up bucket with flag', + { + error, + transientFlag: destinationBucket.hasTransientFlag(), + deletedFlag: destinationBucket.hasDeletedFlag(), + }); + // To avoid confusing user with error + // from cleaning up + // bucket return InternalError + monitoring.promMetrics('PUT', bucketName, 500, 'initiateMultipartUpload'); + return next(errors.InternalError, corsHeaders); + } + return next(null, corsHeaders, destinationBucket); + }); } return next(null, corsHeaders, destinationBucket); }, diff --git a/lib/services.js b/lib/services.js index 966b8d4665..f08a00fc88 100644 --- a/lib/services.js +++ b/lib/services.js @@ -587,6 +587,9 @@ const services = { if (params.legalHold) { multipartObjectMD.legalHold = params.legalHold; } + multipartObjectMD.checksumAlgorithm = params.checksumAlgorithm; + multipartObjectMD.checksumType = params.checksumType; + multipartObjectMD.checksumIsDefault = params.checksumIsDefault; Object.keys(params.metaHeaders).forEach(val => { multipartObjectMD[val] = params.metaHeaders[val]; }); diff --git a/tests/functional/aws-node-sdk/test/object/mpuChecksum.js b/tests/functional/aws-node-sdk/test/object/mpuChecksum.js new file mode 100644 index 0000000000..cb08cda565 --- /dev/null +++ b/tests/functional/aws-node-sdk/test/object/mpuChecksum.js @@ -0,0 +1,178 @@ +const assert = require('assert'); +const { + CreateBucketCommand, + CreateMultipartUploadCommand, + AbortMultipartUploadCommand, + DeleteBucketCommand, +} = require('@aws-sdk/client-s3'); + +const withV4 = require('../support/withV4'); +const BucketUtility = require('../../lib/utility/bucket-util'); + +const bucket = `mpu-checksum-test-${Date.now()}`; +const key = 'test-checksum-key'; + +describe('CreateMultipartUpload checksum headers', () => + withV4(sigCfg => { + let bucketUtil; + let s3; + + before(async () => { + bucketUtil = new BucketUtility('default', sigCfg); + s3 = bucketUtil.s3; + await s3.send(new CreateBucketCommand({ Bucket: bucket })); + }); + + after(async () => { + await bucketUtil.empty(bucket); + await s3.send(new DeleteBucketCommand({ Bucket: bucket })); + }); + + describe('no checksum headers', () => { + let res; + + before(async () => { + res = await s3.send(new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + })); + }); + + after(async () => { + await s3.send(new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: res.UploadId, + })); + }); + + it('should not return ChecksumAlgorithm and ChecksumType', () => { + assert.strictEqual(res.ChecksumAlgorithm, undefined); + assert.strictEqual(res.ChecksumType, undefined); + }); + }); + + describe('valid algorithm only', () => { + const cases = [ + { algo: 'CRC32', expectedType: 'COMPOSITE' }, + { algo: 'CRC32C', expectedType: 'COMPOSITE' }, + { algo: 'CRC64NVME', expectedType: 'FULL_OBJECT' }, + { algo: 'SHA1', expectedType: 'COMPOSITE' }, + { algo: 'SHA256', expectedType: 'COMPOSITE' }, + ]; + + cases.forEach(({ algo, expectedType }) => { + describe(`${algo}`, () => { + let res; + + before(async () => { + res = await s3.send(new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + ChecksumAlgorithm: algo, + })); + }); + + after(async () => { + await s3.send(new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: res.UploadId, + })); + }); + + it(`should return ChecksumAlgorithm ${algo} and ` + + `default ChecksumType to ${expectedType}`, () => { + assert.strictEqual(res.ChecksumAlgorithm, algo); + assert.strictEqual(res.ChecksumType, expectedType); + }); + }); + }); + }); + + describe('valid algorithm + type', () => { + const validCombos = [ + ['CRC32', 'FULL_OBJECT'], + ['CRC32', 'COMPOSITE'], + ['CRC32C', 'FULL_OBJECT'], + ['CRC32C', 'COMPOSITE'], + ['CRC64NVME', 'FULL_OBJECT'], + ['SHA1', 'COMPOSITE'], + ['SHA256', 'COMPOSITE'], + ]; + + validCombos.forEach(([algo, type]) => { + describe(`${algo} + ${type}`, () => { + let res; + + before(async () => { + res = await s3.send(new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + ChecksumAlgorithm: algo, + ChecksumType: type, + })); + }); + + after(async () => { + await s3.send(new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: res.UploadId, + })); + }); + + it(`should return ChecksumAlgorithm ${algo} and ` + + `ChecksumType ${type}`, () => { + assert.strictEqual(res.ChecksumAlgorithm, algo); + assert.strictEqual(res.ChecksumType, type); + }); + }); + }); + }); + + describe('error cases', () => { + it('should reject FULL_OBJECT with SHA256', async () => { + try { + await s3.send(new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + ChecksumAlgorithm: 'SHA256', + ChecksumType: 'FULL_OBJECT', + })); + assert.fail('Expected error'); + } catch (err) { + assert.strictEqual(err.name, 'InvalidRequest'); + } + }); + + it('should reject FULL_OBJECT with SHA1', async () => { + try { + await s3.send(new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + ChecksumAlgorithm: 'SHA1', + ChecksumType: 'FULL_OBJECT', + })); + assert.fail('Expected error'); + } catch (err) { + assert.strictEqual(err.name, 'InvalidRequest'); + } + }); + + it('should reject COMPOSITE with CRC64NVME', async () => { + try { + await s3.send(new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + ChecksumAlgorithm: 'CRC64NVME', + ChecksumType: 'COMPOSITE', + })); + assert.fail('Expected error'); + } catch (err) { + assert.strictEqual(err.name, 'InvalidRequest'); + } + }); + }); + }) +); diff --git a/tests/unit/api/apiUtils/integrity/validateChecksums.js b/tests/unit/api/apiUtils/integrity/validateChecksums.js index d0fc9fec0a..0188ecbc0d 100644 --- a/tests/unit/api/apiUtils/integrity/validateChecksums.js +++ b/tests/unit/api/apiUtils/integrity/validateChecksums.js @@ -9,6 +9,7 @@ const { checksumedMethods, getChecksumDataFromHeaders, arsenalErrorFromChecksumError, + getChecksumDataFromMPUHeaders, } = require('../../../../../lib/api/apiUtils/integrity/validateChecksums'); const { errors: ArsenalErrors } = require('arsenal'); const { config } = require('../../../../../lib/Config'); @@ -732,4 +733,183 @@ describe('arsenalErrorFromChecksumError', () => { const result = arsenalErrorFromChecksumError({ error: 'SomeUnknownError', details: null }); assert.deepStrictEqual(result, ArsenalErrors.BadDigest); }); + + it('should return InvalidRequest for MPUAlgoNotSupported', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.MPUAlgoNotSupported, + details: { algorithm: 'md4' }, + }); + assert.strictEqual(result.message, 'InvalidRequest'); + assert(result.description.includes('[CRC32, CRC32C, CRC64NVME, SHA1, SHA256]')); + }); + + it('should return InvalidRequest for MPUTypeInvalid', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.MPUTypeInvalid, + details: { type: 'BADTYPE' }, + }); + assert.strictEqual(result.message, 'InvalidRequest'); + assert.strictEqual(result.description, + 'Value for x-amz-checksum-type header is invalid.'); + }); + + it('should return InvalidRequest for MPUTypeWithoutAlgo', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.MPUTypeWithoutAlgo, + details: { type: 'COMPOSITE' }, + }); + assert.strictEqual(result.message, 'InvalidRequest'); + assert(result.description.includes( + 'x-amz-checksum-type header can only be used with the x-amz-checksum-algorithm header')); + }); + + it('should return InvalidRequest for MPUInvalidCombination mentioning type and algorithm', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.MPUInvalidCombination, + details: { algorithm: 'sha256', type: 'FULL_OBJECT' }, + }); + assert.strictEqual(result.message, 'InvalidRequest'); + assert.strictEqual(result.description, + 'The FULL_OBJECT checksum type cannot be used with the sha256 checksum algorithm.'); + }); +}); + +describe('getChecksumDataFromMPUHeaders', () => { + describe('no checksum headers (default)', () => { + it('should return crc64nvme/FULL_OBJECT with isDefault=true when no headers', () => { + const result = getChecksumDataFromMPUHeaders({}); + assert.deepStrictEqual(result, { + algorithm: 'crc64nvme', type: 'FULL_OBJECT', isDefault: true, + }); + }); + }); + + describe('algorithm only (no type header)', () => { + const algoDefaults = { + crc32: 'COMPOSITE', + crc32c: 'COMPOSITE', + crc64nvme: 'FULL_OBJECT', + sha1: 'COMPOSITE', + sha256: 'COMPOSITE', + }; + + for (const [algo, expectedType] of Object.entries(algoDefaults)) { + it(`should default to ${expectedType} for ${algo}`, () => { + const result = getChecksumDataFromMPUHeaders({ + 'x-amz-checksum-algorithm': algo, + }); + assert.deepStrictEqual(result, { + algorithm: algo, type: expectedType, isDefault: false, + }); + }); + } + + it('should accept uppercase algorithm (CRC32) and normalize to lowercase', () => { + const result = getChecksumDataFromMPUHeaders({ + 'x-amz-checksum-algorithm': 'CRC32', + }); + assert.deepStrictEqual(result, { + algorithm: 'crc32', type: 'COMPOSITE', isDefault: false, + }); + }); + + it('should accept mixed case algorithm (Sha256) and normalize to lowercase', () => { + const result = getChecksumDataFromMPUHeaders({ + 'x-amz-checksum-algorithm': 'Sha256', + }); + assert.deepStrictEqual(result, { + algorithm: 'sha256', type: 'COMPOSITE', isDefault: false, + }); + }); + }); + + describe('algorithm + type (valid combinations)', () => { + const validCombinations = [ + ['crc32', 'FULL_OBJECT'], + ['crc32', 'COMPOSITE'], + ['crc32c', 'FULL_OBJECT'], + ['crc32c', 'COMPOSITE'], + ['crc64nvme', 'FULL_OBJECT'], + ['sha1', 'COMPOSITE'], + ['sha256', 'COMPOSITE'], + ]; + + for (const [algo, type] of validCombinations) { + it(`should accept ${algo} + ${type}`, () => { + const result = getChecksumDataFromMPUHeaders({ + 'x-amz-checksum-algorithm': algo, + 'x-amz-checksum-type': type, + }); + assert.deepStrictEqual(result, { + algorithm: algo, type, isDefault: false, + }); + }); + } + }); + + describe('unknown algorithm', () => { + it('should return MPUAlgoNotSupported for unknown algorithm', () => { + const result = getChecksumDataFromMPUHeaders({ + 'x-amz-checksum-algorithm': 'md4', + }); + assert.strictEqual(result.error, ChecksumError.MPUAlgoNotSupported); + assert.strictEqual(result.details.algorithm, 'md4'); + }); + + it('should return MPUAlgoNotSupported even when type is also invalid', () => { + const result = getChecksumDataFromMPUHeaders({ + 'x-amz-checksum-algorithm': 'md4', + 'x-amz-checksum-type': 'BADTYPE', + }); + assert.strictEqual(result.error, ChecksumError.MPUAlgoNotSupported); + }); + }); + + describe('unknown type', () => { + it('should return MPUTypeInvalid for unknown type value', () => { + const result = getChecksumDataFromMPUHeaders({ + 'x-amz-checksum-algorithm': 'crc32', + 'x-amz-checksum-type': 'BADTYPE', + }); + assert.strictEqual(result.error, ChecksumError.MPUTypeInvalid); + assert.strictEqual(result.details.type, 'BADTYPE'); + }); + }); + + describe('type without algorithm', () => { + it('should return MPUTypeWithoutAlgo when only type header is sent', () => { + const result = getChecksumDataFromMPUHeaders({ + 'x-amz-checksum-type': 'COMPOSITE', + }); + assert.strictEqual(result.error, ChecksumError.MPUTypeWithoutAlgo); + assert.strictEqual(result.details.type, 'COMPOSITE'); + }); + + it('should return MPUTypeWithoutAlgo even when type value is invalid', () => { + const result = getChecksumDataFromMPUHeaders({ + 'x-amz-checksum-type': 'BADTYPE', + }); + assert.strictEqual(result.error, ChecksumError.MPUTypeWithoutAlgo); + }); + }); + + describe('invalid algorithm + type combinations', () => { + const invalidCombinations = [ + ['sha1', 'FULL_OBJECT'], + ['sha256', 'FULL_OBJECT'], + ['crc64nvme', 'COMPOSITE'], + ]; + + for (const [algo, type] of invalidCombinations) { + it(`should return MPUInvalidCombination for ${algo} + ${type}`, () => { + const result = getChecksumDataFromMPUHeaders({ + 'x-amz-checksum-algorithm': algo, + 'x-amz-checksum-type': type, + }); + assert.strictEqual(result.error, ChecksumError.MPUInvalidCombination); + assert.strictEqual(result.details.algorithm, algo); + assert.strictEqual(result.details.type, type); + }); + } + }); }); diff --git a/tests/unit/api/multipartUpload.js b/tests/unit/api/multipartUpload.js index a2e1198fc4..d052c6b5e5 100644 --- a/tests/unit/api/multipartUpload.js +++ b/tests/unit/api/multipartUpload.js @@ -3295,3 +3295,195 @@ describe('objectPutPart checksum response headers', () => { }); }); }); + +describe('initiateMultipartUpload checksum headers', () => { + const simpleBucketPutRequest = { + bucketName, + namespace, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: '/', + actionImplicitDenies: false, + }; + + beforeEach(done => { + cleanup(); + bucketPut(authInfo, simpleBucketPutRequest, log, done); + }); + + afterEach(() => cleanup()); + + function initiateMPU(headers, cb) { + const request = { + ...initiateRequest, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + ...headers, + }, + }; + initiateMultipartUpload(authInfo, request, log, cb); + } + + function getMPUOverviewMD() { + const mpuKeys = metadata.keyMaps.get(mpuBucket); + const key = mpuKeys.keys().next().value; + return mpuKeys.get(key); + } + + describe('no checksum headers', () => { + it('should not return checksum response headers', done => { + initiateMPU({}, (err, _xml, headers) => { + assert.ifError(err); + assert.strictEqual(headers['x-amz-checksum-algorithm'], undefined); + assert.strictEqual(headers['x-amz-checksum-type'], undefined); + done(); + }); + }); + + it('should store default crc64nvme in MPU metadata', done => { + initiateMPU({}, err => { + assert.ifError(err); + const md = getMPUOverviewMD(); + assert.strictEqual(md.checksumAlgorithm, 'crc64nvme'); + assert.strictEqual(md.checksumType, 'FULL_OBJECT'); + assert.strictEqual(md.checksumIsDefault, true); + done(); + }); + }); + }); + + describe('valid algorithm only', () => { + const cases = [ + { algo: 'CRC32', expectedType: 'COMPOSITE' }, + { algo: 'CRC32C', expectedType: 'COMPOSITE' }, + { algo: 'CRC64NVME', expectedType: 'FULL_OBJECT' }, + { algo: 'SHA1', expectedType: 'COMPOSITE' }, + { algo: 'SHA256', expectedType: 'COMPOSITE' }, + ]; + + cases.forEach(({ algo, expectedType }) => { + it(`should return checksum headers for ${algo}`, done => { + initiateMPU({ 'x-amz-checksum-algorithm': algo }, (err, _xml, headers) => { + assert.ifError(err); + assert.strictEqual(headers['x-amz-checksum-algorithm'], algo); + assert.strictEqual(headers['x-amz-checksum-type'], expectedType); + done(); + }); + }); + + it(`should store ${algo} in MPU metadata with default type ${expectedType}`, done => { + initiateMPU({ 'x-amz-checksum-algorithm': algo }, err => { + assert.ifError(err); + const md = getMPUOverviewMD(); + assert.strictEqual(md.checksumAlgorithm, algo.toLowerCase()); + assert.strictEqual(md.checksumType, expectedType); + assert.strictEqual(md.checksumIsDefault, false); + done(); + }); + }); + }); + + it('should accept lowercase algorithm header', done => { + initiateMPU({ 'x-amz-checksum-algorithm': 'crc32' }, (err, _xml, headers) => { + assert.ifError(err); + assert.strictEqual(headers['x-amz-checksum-algorithm'], 'CRC32'); + assert.strictEqual(headers['x-amz-checksum-type'], 'COMPOSITE'); + done(); + }); + }); + }); + + describe('valid algorithm + type', () => { + const validCombos = [ + ['CRC32', 'FULL_OBJECT'], + ['CRC32', 'COMPOSITE'], + ['CRC32C', 'FULL_OBJECT'], + ['CRC32C', 'COMPOSITE'], + ['CRC64NVME', 'FULL_OBJECT'], + ['SHA1', 'COMPOSITE'], + ['SHA256', 'COMPOSITE'], + ]; + + validCombos.forEach(([algo, type]) => { + it(`should accept ${algo} + ${type}`, done => { + initiateMPU({ + 'x-amz-checksum-algorithm': algo, + 'x-amz-checksum-type': type, + }, (err, _xml, headers) => { + assert.ifError(err); + assert.strictEqual(headers['x-amz-checksum-algorithm'], algo); + assert.strictEqual(headers['x-amz-checksum-type'], type); + done(); + }); + }); + }); + + it('should store checksumIsDefault as false in MPU metadata', done => { + initiateMPU({ + 'x-amz-checksum-algorithm': 'CRC32', + 'x-amz-checksum-type': 'COMPOSITE', + }, err => { + assert.ifError(err); + const md = getMPUOverviewMD(); + assert.strictEqual(md.checksumIsDefault, false); + done(); + }); + }); + }); + + describe('error cases', () => { + it('should reject unknown algorithm', done => { + initiateMPU({ 'x-amz-checksum-algorithm': 'MD4' }, err => { + assert.strictEqual(err.message, 'InvalidRequest'); + done(); + }); + }); + + it('should reject unknown type', done => { + initiateMPU({ + 'x-amz-checksum-algorithm': 'CRC32', + 'x-amz-checksum-type': 'BADTYPE', + }, err => { + assert.strictEqual(err.message, 'InvalidRequest'); + done(); + }); + }); + + it('should reject type without algorithm', done => { + initiateMPU({ 'x-amz-checksum-type': 'COMPOSITE' }, err => { + assert.strictEqual(err.message, 'InvalidRequest'); + done(); + }); + }); + + it('should reject FULL_OBJECT with SHA256', done => { + initiateMPU({ + 'x-amz-checksum-algorithm': 'SHA256', + 'x-amz-checksum-type': 'FULL_OBJECT', + }, err => { + assert.strictEqual(err.message, 'InvalidRequest'); + done(); + }); + }); + + it('should reject COMPOSITE with CRC64NVME', done => { + initiateMPU({ + 'x-amz-checksum-algorithm': 'CRC64NVME', + 'x-amz-checksum-type': 'COMPOSITE', + }, err => { + assert.strictEqual(err.message, 'InvalidRequest'); + done(); + }); + }); + + it('should return algorithm error before type error when both are invalid', done => { + initiateMPU({ + 'x-amz-checksum-algorithm': 'INVALID', + 'x-amz-checksum-type': 'BADTYPE', + }, err => { + assert.strictEqual(err.message, 'InvalidRequest'); + assert(err.description.includes('algorithm')); + done(); + }); + }); + }); +}); diff --git a/tests/unit/lib/services.spec.js b/tests/unit/lib/services.spec.js index 3c7b28fde7..025814bd3d 100644 --- a/tests/unit/lib/services.spec.js +++ b/tests/unit/lib/services.spec.js @@ -3,6 +3,8 @@ const sinon = require('sinon'); const { versioning } = require('arsenal'); const services = require('../../../lib/services'); +const metadata = require('../../../lib/metadata/wrapper'); +const acl = require('../../../lib/metadata/acl'); const { DummyRequestLogger } = require('../helpers'); const { VersionId } = versioning.VersioningConstants; @@ -154,4 +156,82 @@ describe('services', () => { }); }); }); + + describe('metadataStoreMPObject checksum fields', () => { + const baseParams = { + objectKey, + splitter: '|', + uploadId: 'test-upload-id', + eventualStorageBucket: bucketName, + ownerDisplayName: 'owner', + ownerID: 'ownerCanonicalId', + initiatorDisplayName: 'initiator', + initiatorID: 'initiatorId', + headers: {}, + storageClass: 'STANDARD', + metaHeaders: {}, + }; + + let putObjectMDStub; + + beforeEach(() => { + putObjectMDStub = sinon.stub(metadata, 'putObjectMD') + .callsFake((bucket, key, md, opts, reqLog, cb) => cb(null)); + sinon.stub(acl, 'parseAclFromHeaders') + .callsFake((params, cb) => cb(null, { Canned: 'private' })); + }); + + it('should store checksumAlgorithm, checksumType and checksumIsDefault when provided', done => { + const params = { + ...baseParams, + checksumAlgorithm: 'crc32', + checksumType: 'COMPOSITE', + checksumIsDefault: false, + }; + + services.metadataStoreMPObject(bucketName, null, params, log, (err, mpuMD) => { + assert.ifError(err); + assert.strictEqual(mpuMD.checksumAlgorithm, 'crc32'); + assert.strictEqual(mpuMD.checksumType, 'COMPOSITE'); + assert.strictEqual(mpuMD.checksumIsDefault, false); + done(); + }); + }); + + it('should store default crc64nvme with checksumIsDefault true', done => { + const params = { + ...baseParams, + checksumAlgorithm: 'crc64nvme', + checksumType: 'FULL_OBJECT', + checksumIsDefault: true, + }; + + services.metadataStoreMPObject(bucketName, null, params, log, (err, mpuMD) => { + assert.ifError(err); + assert.strictEqual(mpuMD.checksumAlgorithm, 'crc64nvme'); + assert.strictEqual(mpuMD.checksumType, 'FULL_OBJECT'); + assert.strictEqual(mpuMD.checksumIsDefault, true); + done(); + }); + }); + + it('should persist checksum fields to metadata backend', done => { + const params = { + ...baseParams, + checksumAlgorithm: 'sha256', + checksumType: 'COMPOSITE', + checksumIsDefault: false, + }; + + services.metadataStoreMPObject(bucketName, null, params, log, err => { + assert.ifError(err); + sinon.assert.calledOnce(putObjectMDStub); + const storedMD = putObjectMDStub.getCall(0).args[2]; + assert.strictEqual(storedMD.checksumAlgorithm, 'sha256'); + assert.strictEqual(storedMD.checksumType, 'COMPOSITE'); + assert.strictEqual(storedMD.checksumIsDefault, false); + done(); + }); + }); + }); });