Skip to content

Commit 661e8d9

Browse files
leif-scalityleif-scality
committed
CLDSRV-872: store checksum in metadata for objects uploaded with PutObject
1 parent 888ea43 commit 661e8d9

6 files changed

Lines changed: 152 additions & 19 deletions

File tree

‎lib/api/apiUtils/object/createAndStoreObject.js‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -217,7 +217,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
217217
if (size === 0) {
218218
if (!dontSkipBackend[locationType]) {
219219
metadataStoreParams.contentMD5 = constants.emptyFileMd5;
220-
return next(null, null, null);
220+
return next(null, null, null, null);
221221
}
222222

223223
// Handle mdOnlyHeader as a metadata only operation. If
@@ -243,14 +243,14 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
243243
dataStoreVersionId: versionId,
244244
dataStoreMD5: _md5,
245245
};
246-
return next(null, dataGetInfo, _md5);
246+
return next(null, dataGetInfo, _md5, null);
247247
}
248248
}
249249

250250
return dataStore(objectKeyContext, cipherBundle, request, size,
251251
streamingV4Params, backendInfo, log, next);
252252
},
253-
function processDataResult(dataGetInfo, calculatedHash, next) {
253+
function processDataResult(dataGetInfo, calculatedHash, checksum, next) {
254254
if (dataGetInfo === null || dataGetInfo === undefined) {
255255
return next(null, null);
256256
}
@@ -275,6 +275,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
275275
dataGetInfoArr[0].size = mdOnlySize;
276276
}
277277
metadataStoreParams.contentMD5 = calculatedHash;
278+
metadataStoreParams.checksum = checksum;
278279
return next(null, dataGetInfoArr);
279280
},
280281
function getVersioningInfo(infoArr, next) {

‎lib/api/apiUtils/object/storeObject.js‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ const { arsenalErrorFromChecksumError } = require('../../apiUtils/integrity/vali
1717
* @return {function} - calls callback with arguments:
1818
* error, dataRetrievalInfo, and completedHash (if any)
1919
*/
20-
function checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, log, cb) {
20+
function checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, checksumStream, log, cb) {
2121
const contentMD5 = stream.contentMD5;
2222
const completedHash = hashedStream.completedHash;
2323
if (contentMD5 && completedHash && contentMD5 !== completedHash) {
@@ -37,7 +37,10 @@ function checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, log, cb) {
3737
return cb(errors.BadDigest);
3838
});
3939
}
40-
return cb(null, dataRetrievalInfo, completedHash);
40+
const checksum = checksumStream.digest
41+
? { algorithm: checksumStream.algoName, value: checksumStream.digest, type: 'FULL_OBJECT' }
42+
: null;
43+
return cb(null, dataRetrievalInfo, completedHash, checksum);
4144
}
4245

4346
/**
@@ -107,7 +110,8 @@ function dataStore(objectContext, cipherBundle, stream, size,
107110
return cbOnce(arsenalErrorFromChecksumError(checksumErr));
108111
});
109112
}
110-
return checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, log, cbOnce);
113+
return checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo,
114+
checksumedStream.stream, log, cbOnce);
111115
};
112116

113117
// ChecksumTransform._flush computes the digest asynchronously for

‎lib/services.js‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ const { errors, s3middleware } = require('arsenal');
66
const ObjectMD = require('arsenal').models.ObjectMD;
77
const BucketInfo = require('arsenal').models.BucketInfo;
88
const ObjectMDArchive = require('arsenal').models.ObjectMDArchive;
9+
const ObjectMDChecksum = require('arsenal').models.ObjectMDChecksum;
910
const { versioning } = require('arsenal');
1011
const acl = require('./metadata/acl');
1112
const constants = require('../constants');
@@ -102,7 +103,7 @@ const services = {
102103
* @return {function} executes callback with err or ETag as arguments
103104
*/
104105
metadataStoreObject(bucketName, dataGetInfo, cipherBundle, params, cb) {
105-
const { objectKey, authInfo, size, contentMD5, metaHeaders,
106+
const { objectKey, authInfo, size, contentMD5, checksum, metaHeaders,
106107
contentType, cacheControl, contentDisposition, contentEncoding,
107108
expires, multipart, headers, overrideMetadata, log,
108109
lastModifiedDate, versioning, versionId, uploadId,
@@ -138,6 +139,9 @@ const services = {
138139
// CreationTime needs to be carried over so that it remains static
139140
.setCreationTime(creationTime)
140141
.setOriginOp(originOp);
142+
if (checksum) {
143+
md.setChecksum(new ObjectMDChecksum(checksum.algorithm, checksum.value, checksum.type));
144+
}
141145
// Sending in last modified date in object put copy since need
142146
// to return the exact date in the response
143147
if (lastModifiedDate) {

‎tests/functional/aws-node-sdk/test/object/putVersion.js‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -258,7 +258,7 @@ describe('PUT object with x-scal-s3-version-id header', () => {
258258
assert.deepStrictEqual(versionsAfter, versionsBefore);
259259

260260
checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length',
261-
'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'originOp']);
261+
'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'originOp', 'checksum']);
262262
assert.deepStrictEqual(objMDAfter, objMDBefore);
263263
return done();
264264
});
@@ -309,7 +309,7 @@ describe('PUT object with x-scal-s3-version-id header', () => {
309309
assert.deepStrictEqual(versionsAfter, versionsBefore);
310310

311311
checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp',
312-
'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']);
312+
'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']);
313313
assert.deepStrictEqual(objMDAfter, objMDBefore);
314314
return done();
315315
});
@@ -360,7 +360,7 @@ describe('PUT object with x-scal-s3-version-id header', () => {
360360
assert.deepStrictEqual(versionsAfter, versionsBefore);
361361

362362
checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp',
363-
'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']);
363+
'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']);
364364
assert.deepStrictEqual(objMDAfter, objMDBefore);
365365
return done();
366366
});
@@ -408,7 +408,7 @@ describe('PUT object with x-scal-s3-version-id header', () => {
408408
assert.deepStrictEqual(versionsAfter, versionsBefore);
409409

410410
checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp',
411-
'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']);
411+
'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']);
412412
assert.deepStrictEqual(objMDAfter, objMDBefore);
413413
return done();
414414
});
@@ -460,7 +460,7 @@ describe('PUT object with x-scal-s3-version-id header', () => {
460460
assert.deepStrictEqual(versionsAfter, versionsBefore);
461461

462462
checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp',
463-
'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']);
463+
'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']);
464464
assert.deepStrictEqual(objMDAfter, objMDBefore);
465465
return done();
466466
});
@@ -515,7 +515,7 @@ describe('PUT object with x-scal-s3-version-id header', () => {
515515
assert.deepStrictEqual(versionsAfter, versionsBefore);
516516

517517
checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp',
518-
'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']);
518+
'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']);
519519
assert.deepStrictEqual(objMDAfter, objMDBefore);
520520
return done();
521521
});
@@ -568,7 +568,7 @@ describe('PUT object with x-scal-s3-version-id header', () => {
568568
assert.deepStrictEqual(versionsAfter, versionsBefore);
569569

570570
checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp',
571-
'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']);
571+
'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']);
572572
assert.deepStrictEqual(objMDAfter, objMDBefore);
573573
return done();
574574
});
@@ -620,7 +620,7 @@ describe('PUT object with x-scal-s3-version-id header', () => {
620620
assert.deepStrictEqual(versionsAfter, versionsBefore);
621621

622622
checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp',
623-
'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']);
623+
'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']);
624624
assert.deepStrictEqual(objMDAfter, objMDBefore);
625625
return done();
626626
});
@@ -679,7 +679,7 @@ describe('PUT object with x-scal-s3-version-id header', () => {
679679
assert.deepStrictEqual(versionsAfter, versionsBefore);
680680

681681
checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp',
682-
'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']);
682+
'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']);
683683
assert.deepStrictEqual(objMDAfter, objMDBefore);
684684
return done();
685685
});
@@ -726,7 +726,7 @@ describe('PUT object with x-scal-s3-version-id header', () => {
726726
assert.deepStrictEqual(versionsAfter, versionsBefore);
727727

728728
checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp',
729-
'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']);
729+
'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']);
730730
assert.deepStrictEqual(objMDAfter, objMDBefore);
731731
return done();
732732
});

‎tests/functional/raw-node/test/routes/routeMetadata.js‎

Lines changed: 79 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
const assert = require('assert');
2+
const crypto = require('crypto');
23
const http = require('http');
3-
const { CreateBucketCommand,
4-
PutObjectCommand,
4+
const { CreateBucketCommand,
5+
PutObjectCommand,
6+
DeleteObjectCommand,
57
DeleteBucketCommand } = require('@aws-sdk/client-s3');
68

79
const { makeRequest } = require('../../utils/makeRequest');
@@ -144,3 +146,78 @@ describe('metadata routes with metadata', () => {
144146
});
145147
});
146148
});
149+
150+
describe('checksum stored in object metadata after PutObject', () => {
151+
const bucketUtil = new BucketUtility('default', { signatureVersion: 'v4' });
152+
const s3 = bucketUtil.s3;
153+
154+
const bucket = 'bucket-checksum-test';
155+
const objectBody = 'hello checksum';
156+
const sha256Key = 'object-with-sha256-checksum';
157+
const defaultKey = 'object-with-default-checksum';
158+
159+
// CRC32 of 'hello checksum' in base64 — the AWS SDK v3 injects x-amz-checksum-crc32 by default
160+
const expectedCrc32 = 'EyV5Tg==';
161+
162+
before(async function () {
163+
if (!process.env.S3_END_TO_END) {
164+
this.skip();
165+
}
166+
await s3.send(new CreateBucketCommand({ Bucket: bucket }));
167+
168+
const sha256Value = crypto.createHash('sha256').update(objectBody).digest('base64');
169+
await s3.send(new PutObjectCommand({
170+
Bucket: bucket,
171+
Key: sha256Key,
172+
Body: objectBody,
173+
ChecksumSHA256: sha256Value,
174+
}));
175+
await s3.send(new PutObjectCommand({
176+
Bucket: bucket,
177+
Key: defaultKey,
178+
Body: objectBody,
179+
}));
180+
});
181+
182+
after(async () => {
183+
if (!process.env.S3_END_TO_END) {
184+
return;
185+
}
186+
await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: sha256Key }));
187+
await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: defaultKey }));
188+
await s3.send(new DeleteBucketCommand({ Bucket: bucket }));
189+
});
190+
191+
it('stores sha256 checksum in metadata when x-amz-checksum-sha256 is provided', done => {
192+
const expectedValue = crypto.createHash('sha256').update(objectBody).digest('base64');
193+
makeMetadataRequest({
194+
method: 'GET',
195+
authCredentials: metadataAuthCredentials,
196+
path: `/_/metadata/default/bucket/${bucket}/${sha256Key}`,
197+
}, (err, res) => {
198+
assert.ifError(err);
199+
assert.strictEqual(res.statusCode, 200);
200+
const md = JSON.parse(res.body);
201+
assert.strictEqual(md.checksum.checksumAlgorithm, 'sha256');
202+
assert.strictEqual(md.checksum.checksumValue, expectedValue);
203+
assert.strictEqual(md.checksum.checksumType, 'FULL_OBJECT');
204+
return done();
205+
});
206+
});
207+
208+
it('stores crc32 checksum in metadata when no explicit checksum header is provided (AWS SDK default)', done => {
209+
makeMetadataRequest({
210+
method: 'GET',
211+
authCredentials: metadataAuthCredentials,
212+
path: `/_/metadata/default/bucket/${bucket}/${defaultKey}`,
213+
}, (err, res) => {
214+
assert.ifError(err);
215+
assert.strictEqual(res.statusCode, 200);
216+
const md = JSON.parse(res.body);
217+
assert.strictEqual(md.checksum.checksumAlgorithm, 'crc32');
218+
assert.strictEqual(md.checksum.checksumValue, expectedCrc32);
219+
assert.strictEqual(md.checksum.checksumType, 'FULL_OBJECT');
220+
return done();
221+
});
222+
});
223+
});

‎tests/unit/api/objectPut.js‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
const assert = require('assert');
22
const async = require('async');
3+
const crypto = require('crypto');
34
const moment = require('moment');
45
const { s3middleware, storage, versioning } = require('arsenal');
56
const sinon = require('sinon');
@@ -865,6 +866,52 @@ describe('objectPut API', () => {
865866
});
866867
});
867868
});
869+
870+
it('should store sha256 checksum in metadata when x-amz-checksum-sha256 header is provided', done => {
871+
const sha256Value = crypto.createHash('sha256').update(postBody).digest('base64');
872+
const request = new DummyRequest({
873+
bucketName,
874+
namespace,
875+
objectKey: objectName,
876+
headers: {
877+
host: `${bucketName}.s3.amazonaws.com`,
878+
'x-amz-checksum-sha256': sha256Value,
879+
},
880+
url: '/',
881+
}, postBody);
882+
883+
bucketPut(authInfo, testPutBucketRequest, log, err => {
884+
assert.ifError(err);
885+
objectPut(authInfo, request, undefined, log, err => {
886+
assert.ifError(err);
887+
metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => {
888+
assert.ifError(err);
889+
assert(md.checksum, 'checksum should be set in metadata');
890+
assert.strictEqual(md.checksum.checksumAlgorithm, 'sha256');
891+
assert.strictEqual(md.checksum.checksumValue, sha256Value);
892+
assert.strictEqual(md.checksum.checksumType, 'FULL_OBJECT');
893+
done();
894+
});
895+
});
896+
});
897+
});
898+
899+
it('should store crc64nvme checksum in metadata when no checksum header is provided', done => {
900+
bucketPut(authInfo, testPutBucketRequest, log, err => {
901+
assert.ifError(err);
902+
objectPut(authInfo, testPutObjectRequest, undefined, log, err => {
903+
assert.ifError(err);
904+
metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => {
905+
assert.ifError(err);
906+
assert(md.checksum, 'checksum should be set in metadata');
907+
assert.strictEqual(md.checksum.checksumAlgorithm, 'crc64nvme');
908+
assert(md.checksum.checksumValue, 'checksumValue should be set');
909+
assert.strictEqual(md.checksum.checksumType, 'FULL_OBJECT');
910+
done();
911+
});
912+
});
913+
});
914+
});
868915
});
869916

870917
describe('objectPut API with versioning', () => {

0 commit comments

Comments
 (0)