|
| 1 | +const assert = require('assert'); |
| 2 | +const async = require('async'); |
| 3 | +const crypto = require('crypto'); |
| 4 | +const { storage } = require('arsenal'); |
| 5 | +const { parseString } = require('xml2js'); |
| 6 | + |
| 7 | +const { bucketPut } = require('../../../lib/api/bucketPut'); |
| 8 | +const initiateMultipartUpload = require('../../../lib/api/initiateMultipartUpload'); |
| 9 | +const objectPutPart = require('../../../lib/api/objectPutPart'); |
| 10 | +const constants = require('../../../constants'); |
| 11 | +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); |
| 12 | +const DummyRequest = require('../DummyRequest'); |
| 13 | + |
| 14 | +const { metadata } = storage.metadata.inMemory.metadata; |
| 15 | + |
| 16 | +const log = new DummyRequestLogger(); |
| 17 | +const splitter = constants.splitter; |
| 18 | +const canonicalID = 'accessKey1'; |
| 19 | +const authInfo = makeAuthInfo(canonicalID); |
| 20 | +const namespace = 'default'; |
| 21 | +const bucketName = 'checksum-test-bucket'; |
| 22 | +const objectKey = 'testObject'; |
| 23 | +const mpuBucket = `${constants.mpuBucketPrefix}${bucketName}`; |
| 24 | +const partBody = Buffer.from('I am a part body for checksum testing', 'utf8'); |
| 25 | + |
| 26 | +const bucketPutRequest = { |
| 27 | + bucketName, |
| 28 | + namespace, |
| 29 | + headers: { host: `${bucketName}.s3.amazonaws.com` }, |
| 30 | + url: '/', |
| 31 | + actionImplicitDenies: false, |
| 32 | +}; |
| 33 | + |
| 34 | +function makeInitiateRequest(extraHeaders = {}) { |
| 35 | + return { |
| 36 | + socket: { remoteAddress: '1.1.1.1' }, |
| 37 | + bucketName, |
| 38 | + namespace, |
| 39 | + objectKey, |
| 40 | + headers: { |
| 41 | + host: `${bucketName}.s3.amazonaws.com`, |
| 42 | + ...extraHeaders, |
| 43 | + }, |
| 44 | + url: `/${objectKey}?uploads`, |
| 45 | + actionImplicitDenies: false, |
| 46 | + }; |
| 47 | +} |
| 48 | + |
| 49 | +function makePutPartRequest(uploadId, partNumber, body, extraHeaders = {}) { |
| 50 | + const md5Hash = crypto.createHash('md5').update(body); |
| 51 | + return new DummyRequest({ |
| 52 | + bucketName, |
| 53 | + namespace, |
| 54 | + objectKey, |
| 55 | + headers: { |
| 56 | + host: `${bucketName}.s3.amazonaws.com`, |
| 57 | + ...extraHeaders, |
| 58 | + }, |
| 59 | + url: `/${objectKey}?partNumber=${partNumber}&uploadId=${uploadId}`, |
| 60 | + query: { partNumber, uploadId }, |
| 61 | + partHash: md5Hash.digest('hex'), |
| 62 | + actionImplicitDenies: false, |
| 63 | + }, body); |
| 64 | +} |
| 65 | + |
| 66 | +function initiateMPU(initiateHeaders, cb) { |
| 67 | + async.waterfall([ |
| 68 | + next => bucketPut(authInfo, bucketPutRequest, log, next), |
| 69 | + (corsHeaders, next) => { |
| 70 | + const req = makeInitiateRequest(initiateHeaders); |
| 71 | + initiateMultipartUpload(authInfo, req, log, next); |
| 72 | + }, |
| 73 | + (result, corsHeaders, next) => parseString(result, next), |
| 74 | + ], (err, json) => { |
| 75 | + if (err) return cb(err); |
| 76 | + return cb(null, json.InitiateMultipartUploadResult.UploadId[0]); |
| 77 | + }); |
| 78 | +} |
| 79 | + |
| 80 | +function getPartMetadata(uploadId) { |
| 81 | + const mpuKeys = metadata.keyMaps.get(mpuBucket); |
| 82 | + if (!mpuKeys) return null; |
| 83 | + for (const [key, val] of mpuKeys) { |
| 84 | + if (key.startsWith(uploadId) && !key.startsWith('overview')) { |
| 85 | + return val; |
| 86 | + } |
| 87 | + } |
| 88 | + return null; |
| 89 | +} |
| 90 | + |
| 91 | +describe('objectPutPart checksum validation', () => { |
| 92 | + beforeEach(() => cleanup()); |
| 93 | + |
| 94 | + describe('algo match validation', () => { |
| 95 | + it('should accept part with matching checksum algo', done => { |
| 96 | + initiateMPU({ 'x-amz-checksum-algorithm': 'crc32' }, (err, uploadId) => { |
| 97 | + assert.ifError(err); |
| 98 | + const request = makePutPartRequest(uploadId, 1, partBody, { |
| 99 | + 'x-amz-checksum-crc32': 'AAAAAA==', |
| 100 | + }); |
| 101 | + objectPutPart(authInfo, request, undefined, log, err => { |
| 102 | + // BadDigest is expected since the checksum value won't |
| 103 | + // match the body, but NOT InvalidRequest — the algo is accepted. |
| 104 | + if (err) { |
| 105 | + assert.notStrictEqual(err.message, 'InvalidRequest'); |
| 106 | + } |
| 107 | + done(); |
| 108 | + }); |
| 109 | + }); |
| 110 | + }); |
| 111 | + |
| 112 | + it('should reject part with mismatching checksum algo', done => { |
| 113 | + initiateMPU({ 'x-amz-checksum-algorithm': 'sha256' }, (err, uploadId) => { |
| 114 | + assert.ifError(err); |
| 115 | + const request = makePutPartRequest(uploadId, 1, partBody, { |
| 116 | + 'x-amz-checksum-crc32': 'AAAAAA==', |
| 117 | + }); |
| 118 | + objectPutPart(authInfo, request, undefined, log, err => { |
| 119 | + assert(err, 'Expected an error'); |
| 120 | + assert.strictEqual(err.message, 'InvalidRequest'); |
| 121 | + done(); |
| 122 | + }); |
| 123 | + }); |
| 124 | + }); |
| 125 | + |
| 126 | + it('should accept part with no checksum on non-default MPU', done => { |
| 127 | + initiateMPU({ 'x-amz-checksum-algorithm': 'sha256' }, (err, uploadId) => { |
| 128 | + assert.ifError(err); |
| 129 | + // No checksum header sent |
| 130 | + const request = makePutPartRequest(uploadId, 1, partBody); |
| 131 | + objectPutPart(authInfo, request, undefined, log, err => { |
| 132 | + assert.ifError(err); |
| 133 | + done(); |
| 134 | + }); |
| 135 | + }); |
| 136 | + }); |
| 137 | + |
| 138 | + it('should accept any checksum algo on default (no algo specified) MPU', done => { |
| 139 | + initiateMPU({}, (err, uploadId) => { |
| 140 | + assert.ifError(err); |
| 141 | + // Send sha256 checksum even though MPU is default crc64nvme |
| 142 | + const request = makePutPartRequest(uploadId, 1, partBody, { |
| 143 | + 'x-amz-checksum-sha256': 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=', |
| 144 | + }); |
| 145 | + objectPutPart(authInfo, request, undefined, log, err => { |
| 146 | + // BadDigest (wrong value) is fine; InvalidRequest (wrong algo) is not |
| 147 | + if (err) { |
| 148 | + assert.notStrictEqual(err.message, 'InvalidRequest'); |
| 149 | + } |
| 150 | + done(); |
| 151 | + }); |
| 152 | + }); |
| 153 | + }); |
| 154 | + }); |
| 155 | + |
| 156 | + describe('checksum stored in part metadata', () => { |
| 157 | + it('should store checksumValue and checksumAlgorithm in part metadata', done => { |
| 158 | + initiateMPU({}, (err, uploadId) => { |
| 159 | + assert.ifError(err); |
| 160 | + const request = makePutPartRequest(uploadId, 1, partBody); |
| 161 | + objectPutPart(authInfo, request, undefined, log, err => { |
| 162 | + assert.ifError(err); |
| 163 | + const partMD = getPartMetadata(uploadId); |
| 164 | + assert(partMD, 'Part metadata should exist'); |
| 165 | + assert(partMD.checksumValue, 'checksumValue should be stored'); |
| 166 | + assert.strictEqual(partMD.checksumAlgorithm, 'crc64nvme'); |
| 167 | + done(); |
| 168 | + }); |
| 169 | + }); |
| 170 | + }); |
| 171 | + |
| 172 | + it('should store the MPU algo checksum when client sends matching algo', done => { |
| 173 | + initiateMPU({ 'x-amz-checksum-algorithm': 'crc64nvme' }, (err, uploadId) => { |
| 174 | + assert.ifError(err); |
| 175 | + const request = makePutPartRequest(uploadId, 1, partBody); |
| 176 | + objectPutPart(authInfo, request, undefined, log, err => { |
| 177 | + assert.ifError(err); |
| 178 | + const partMD = getPartMetadata(uploadId); |
| 179 | + assert(partMD); |
| 180 | + assert.strictEqual(partMD.checksumAlgorithm, 'crc64nvme'); |
| 181 | + assert(partMD.checksumValue); |
| 182 | + done(); |
| 183 | + }); |
| 184 | + }); |
| 185 | + }); |
| 186 | + }); |
| 187 | + |
| 188 | + describe('dual-checksum', () => { |
| 189 | + it('should store crc64nvme when default MPU and client sends different algo', done => { |
| 190 | + initiateMPU({}, (err, uploadId) => { |
| 191 | + assert.ifError(err); |
| 192 | + // Compute correct crc32 for partBody so validation passes |
| 193 | + const { algorithms } = require('../../../lib/api/apiUtils/integrity/validateChecksums'); |
| 194 | + const crc32Hash = algorithms.crc32.createHash(); |
| 195 | + crc32Hash.update(partBody); |
| 196 | + const crc64Hash = algorithms.crc64nvme.createHash(); |
| 197 | + crc64Hash.update(partBody); |
| 198 | + Promise.all([ |
| 199 | + algorithms.crc32.digestFromHash(crc32Hash), |
| 200 | + algorithms.crc64nvme.digestFromHash(crc64Hash), |
| 201 | + ]).then(([crc32Digest, crc64Digest]) => { |
| 202 | + const request = makePutPartRequest(uploadId, 1, partBody, { |
| 203 | + 'x-amz-checksum-crc32': crc32Digest, |
| 204 | + }); |
| 205 | + objectPutPart(authInfo, request, undefined, log, (err, hexDigest, corsHeaders) => { |
| 206 | + assert.ifError(err); |
| 207 | + // Response header should be the client's algo (crc32) |
| 208 | + assert.strictEqual(corsHeaders['x-amz-checksum-crc32'], crc32Digest); |
| 209 | + // Stored metadata should be crc64nvme with correct value |
| 210 | + const partMD = getPartMetadata(uploadId); |
| 211 | + assert(partMD); |
| 212 | + assert.strictEqual(partMD.checksumAlgorithm, 'crc64nvme'); |
| 213 | + assert.strictEqual(partMD.checksumValue, crc64Digest); |
| 214 | + done(); |
| 215 | + }); |
| 216 | + }); |
| 217 | + }); |
| 218 | + }); |
| 219 | + |
| 220 | + it('should handle dual-checksum with trailer (STREAMING-UNSIGNED-PAYLOAD-TRAILER)', done => { |
| 221 | + initiateMPU({}, (err, uploadId) => { |
| 222 | + assert.ifError(err); |
| 223 | + const { algorithms } = require('../../../lib/api/apiUtils/integrity/validateChecksums'); |
| 224 | + const hash = algorithms.sha256.createHash(); |
| 225 | + hash.update(partBody); |
| 226 | + const crc64Hash = algorithms.crc64nvme.createHash(); |
| 227 | + crc64Hash.update(partBody); |
| 228 | + Promise.all([ |
| 229 | + algorithms.sha256.digestFromHash(hash), |
| 230 | + algorithms.crc64nvme.digestFromHash(crc64Hash), |
| 231 | + ]).then(([sha256Digest, crc64Digest]) => { |
| 232 | + // Build chunked body with trailing checksum |
| 233 | + const hexLen = partBody.length.toString(16); |
| 234 | + const chunkedBody = `${hexLen}\r\n${partBody.toString()}\r\n` + |
| 235 | + `0\r\nx-amz-checksum-sha256:${sha256Digest}\r\n`; |
| 236 | + const request = makePutPartRequest(uploadId, 1, Buffer.from(chunkedBody), { |
| 237 | + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', |
| 238 | + 'x-amz-trailer': 'x-amz-checksum-sha256', |
| 239 | + }); |
| 240 | + request.parsedContentLength = partBody.length; |
| 241 | + objectPutPart(authInfo, request, undefined, log, (err, hexDigest, corsHeaders) => { |
| 242 | + assert.ifError(err); |
| 243 | + // Response should echo the client's sha256 |
| 244 | + assert.strictEqual(corsHeaders['x-amz-checksum-sha256'], sha256Digest); |
| 245 | + // Stored metadata should be crc64nvme with correct value |
| 246 | + const partMD = getPartMetadata(uploadId); |
| 247 | + assert(partMD); |
| 248 | + assert.strictEqual(partMD.checksumAlgorithm, 'crc64nvme'); |
| 249 | + assert.strictEqual(partMD.checksumValue, crc64Digest); |
| 250 | + done(); |
| 251 | + }); |
| 252 | + }); |
| 253 | + }); |
| 254 | + }); |
| 255 | + |
| 256 | + it('should return client-facing checksum in response header for dual-checksum', done => { |
| 257 | + initiateMPU({}, (err, uploadId) => { |
| 258 | + assert.ifError(err); |
| 259 | + const { algorithms } = require('../../../lib/api/apiUtils/integrity/validateChecksums'); |
| 260 | + const hash = algorithms.sha256.createHash(); |
| 261 | + hash.update(partBody); |
| 262 | + Promise.resolve(algorithms.sha256.digestFromHash(hash)).then(digest => { |
| 263 | + const request = makePutPartRequest(uploadId, 1, partBody, { |
| 264 | + 'x-amz-checksum-sha256': digest, |
| 265 | + }); |
| 266 | + objectPutPart(authInfo, request, undefined, log, (err, hexDigest, corsHeaders) => { |
| 267 | + assert.ifError(err); |
| 268 | + assert.strictEqual(corsHeaders['x-amz-checksum-sha256'], digest); |
| 269 | + assert.strictEqual(corsHeaders['x-amz-checksum-crc64nvme'], undefined); |
| 270 | + done(); |
| 271 | + }); |
| 272 | + }); |
| 273 | + }); |
| 274 | + }); |
| 275 | + }); |
| 276 | +}); |
0 commit comments