CLDSRV-863: handle trailer headers and add new trailer errors

Author: leif-scality

lib/api/apiUtils/integrity/validateChecksums.js

@@ -37,6 +37,12 @@ const ChecksumError = Object.freeze({
     MultipleChecksumTypes: 'MultipleChecksumTypes',
     MissingCorresponding: 'MissingCorresponding',
     MalformedChecksum: 'MalformedChecksum',
+    TrailerAlgoMismatch: 'TrailerAlgoMismatch',
+    TrailerChecksumMalformed: 'TrailerChecksumMalformed',
+    TrailerMissing: 'TrailerMissing',
+    TrailerUnexpected: 'TrailerUnexpected',
+    TrailerAndChecksum: 'TrailerAndChecksum',
+    TrailerNotSupported: 'TrailerNotSupported',
 });
 
 const base64Regex = /^[A-Za-z0-9+/]*={0,2}$/;
@@ -56,35 +62,51 @@ const algorithms = Object.freeze({
             const result = await crc.digest();
             return Buffer.from(result).toString('base64');
         },
+        digestFromHash: async hash => {
+            const result = await hash.digest();
+            return Buffer.from(result).toString('base64');
+        },
         isValidDigest: expected => typeof expected === 'string' && expected.length === 12 && base64Regex.test(expected),
+        createHash: () => new CrtCrc64Nvme()
     },
     crc32: {
         digest: data => {
             const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
             return uint32ToBase64(new Crc32().update(input).digest() >>> 0); // >>> 0 coerce number to uint32
         },
+        digestFromHash: hash => {
+            const result = hash.digest();
+            return uint32ToBase64(result >>> 0);
+        },
         isValidDigest: expected => typeof expected === 'string' && expected.length === 8 && base64Regex.test(expected),
+        createHash: () => new Crc32()
     },
     crc32c: {
         digest: data => {
             const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
             return uint32ToBase64(new Crc32c().update(input).digest() >>> 0); // >>> 0 coerce number to uint32
         },
+        digestFromHash: hash => uint32ToBase64(hash.digest() >>> 0),
         isValidDigest: expected => typeof expected === 'string' && expected.length === 8 && base64Regex.test(expected),
+        createHash: () => new Crc32c()
     },
     sha1: {
         digest: data => {
             const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
             return crypto.createHash('sha1').update(input).digest('base64');
         },
+        digestFromHash: hash => hash.digest('base64'),
         isValidDigest: expected => typeof expected === 'string' && expected.length === 28 && base64Regex.test(expected),
+        createHash: () => crypto.createHash('sha1')
     },
     sha256: {
         digest: data => {
             const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
             return crypto.createHash('sha256').update(input).digest('base64');
         },
+        digestFromHash: hash => hash.digest('base64'),
         isValidDigest: expected => typeof expected === 'string' && expected.length === 44 && base64Regex.test(expected),
+        createHash: () => crypto.createHash('sha256')
     }
 });
 
@@ -141,6 +163,94 @@ async function validateXAmzChecksums(headers, body) {
     return null;
 }
 
+function getChecksumDataFromHeaders(headers) {
+    const checkSdk = algo => {
+        if (!('x-amz-sdk-checksum-algorithm' in headers)) {
+            return null;
+        }
+
+        const sdkAlgo = headers['x-amz-sdk-checksum-algorithm'];
+        if (typeof sdkAlgo !== 'string') {
+            return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } };
+        }
+
+        const sdkLowerAlgo = sdkAlgo.toLowerCase();
+        if (!(sdkLowerAlgo in algorithms)) {
+            return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } };
+        }
+
+        // If AWS there is a mismatch, AWS returns the same error as if the algo was invalid.
+        if (sdkLowerAlgo !== algo) {
+            return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } };
+        }
+
+        return null;
+    };
+
+    const checksumHeaders = Object.keys(headers).filter(header => header.startsWith('x-amz-checksum-'));
+    const xAmzChecksumCnt = checksumHeaders.length;
+    if (xAmzChecksumCnt > 1) {
+        return { error: ChecksumError.MultipleChecksumTypes, details: { algorithms: checksumHeaders } };
+    }
+
+    if (xAmzChecksumCnt === 0 && !('x-amz-trailer' in headers) && 'x-amz-sdk-checksum-algorithm' in headers) {
+        return {
+            error: ChecksumError.MissingCorresponding,
+            details: { expected: headers['x-amz-sdk-checksum-algorithm'] }
+        };
+    }
+
+    if ('x-amz-trailer' in headers) {
+        if (xAmzChecksumCnt !== 0) {
+            return {
+                error: ChecksumError.TrailerAndChecksum,
+                details: { trailer: headers['x-amz-trailer'], checksum: checksumHeaders },
+            };
+        }
+
+        const trailer = headers['x-amz-trailer'];
+        if (!trailer.startsWith('x-amz-checksum-')) {
+            return { error: ChecksumError.TrailerNotSupported, details: { value: trailer } };
+        }
+
+        const trailerAlgo = trailer.slice('x-amz-checksum-'.length);
+        if (!(trailerAlgo in algorithms)) {
+            return { error: ChecksumError.TrailerNotSupported, details: { value: trailer } };
+        }
+
+        const err = checkSdk(trailerAlgo);
+        if (err) {
+            return err;
+        }
+
+        return { algorithm: trailerAlgo, isTrailer: true, expected: undefined };
+    }
+
+    if (xAmzChecksumCnt === 0) {
+        // There was no x-amz-checksum- or x-amz-trailer return crc64nvme.
+        // The calculated crc64nvme will be stored in the object metadata.
+        return { algorithm: 'crc64nvme', isTrailer: false, expected: undefined };
+    }
+
+    // No x-amz-sdk-checksum-algorithm we expect one x-amz-checksum-[crc64nvme, crc32, crc32C, sha1, sha256].
+    const algo = checksumHeaders[0].slice('x-amz-checksum-'.length);
+    if (!(algo in algorithms)) {
+        return { error: ChecksumError.AlgoNotSupported, details: { algorithm: algo } };
+    }
+
+    const expected = headers[`x-amz-checksum-${algo}`];
+    if (!algorithms[algo].isValidDigest(expected)) {
+        return { error: ChecksumError.MalformedChecksum, details: { algorithm: algo, expected } };
+    }
+
+    const err = checkSdk(algo);
+    if (err) {
+        return err;
+    }
+
+    return { algorithm: algo, isTrailer: false, expected };
+}
+
 /**
  * validateChecksumsNoChunking - Validate the checksums of a request.
  * @param {object} headers - http headers
@@ -183,16 +293,7 @@ async function validateChecksumsNoChunking(headers, body) {
     return err;
 }
 
-async function defaultValidationFunc(request, body, log) {
-    const err = await validateChecksumsNoChunking(request.headers, body);
-    if (!err) {
-        return null;
-    }
-
-    if (err.error !== ChecksumError.MissingChecksum) {
-        log.debug('failed checksum validation', { method: request.apiMethod }, err);
-    }
-
+function arsenalErrorFromChecksumError(err) {
     switch (err.error) {
         case ChecksumError.MissingChecksum:
             return null;
@@ -225,11 +326,40 @@ async function defaultValidationFunc(request, body, log) {
             );
         case ChecksumError.MD5Invalid:
             return ArsenalErrors.InvalidDigest;
+        case ChecksumError.TrailerAlgoMismatch:
+            return ArsenalErrors.MalformedTrailerError;
+        case ChecksumError.TrailerMissing:
+            return ArsenalErrors.MalformedTrailerError;
+        case ChecksumError.TrailerUnexpected:
+            return ArsenalErrors.MalformedTrailerError;
+        case ChecksumError.TrailerChecksumMalformed:
+            return ArsenalErrors.InvalidRequest.customizeDescription(
+                `Value for x-amz-checksum-${err.details.algorithm} trailing header is invalid.`
+            );
+        case ChecksumError.TrailerAndChecksum:
+            return ArsenalErrors.InvalidRequest.customizeDescription('Expecting a single x-amz-checksum- header');
+        case ChecksumError.TrailerNotSupported:
+            return ArsenalErrors.InvalidRequest.customizeDescription(
+                'The value specified in the x-amz-trailer header is not supported'
+            );
         default:
             return ArsenalErrors.BadDigest;
     }
 }
 
+async function defaultValidationFunc(request, body, log) {
+    const err = await validateChecksumsNoChunking(request.headers, body);
+    if (!err) {
+        return null;
+    }
+
+    if (err.error !== ChecksumError.MissingChecksum) {
+        log.debug('failed checksum validation', { method: request.apiMethod }, err);
+    }
+
+    return arsenalErrorFromChecksumError(err);
+}
+
 /**
  * validateMethodChecksumsNoChunking - Validate the checksums of a request.
  * @param {object} request - http request
@@ -253,5 +383,8 @@ module.exports = {
     ChecksumError,
     validateChecksumsNoChunking,
     validateMethodChecksumNoChunking,
+    getChecksumDataFromHeaders,
+    arsenalErrorFromChecksumError,
+    algorithms,
     checksumedMethods,
 };