Skip to content

Commit 541bec8

Browse files
leif-scalityleif-scality
committed
CLDSRV-873: return checksum from GetObject
1 parent d9fe250 commit 541bec8

File tree

6 files changed

+309
-12
lines changed

6 files changed

+309
-12
lines changed

‎lib/api/objectGet.js‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,12 @@ function objectGet(authInfo, request, returnTagCount, log, callback) {
4141
const bucketName = request.bucketName;
4242
const objectKey = request.objectKey;
4343

44+
const checksumMode = request.headers['x-amz-checksum-mode'];
45+
if (checksumMode !== undefined && checksumMode !== 'ENABLED') {
46+
log.debug('invalid x-amz-checksum-mode', { checksumMode });
47+
return callback(errors.InvalidArgument);
48+
}
49+
4450
// returns name of location to get from and key if successful
4551
const locCheckResult =
4652
locationHeaderCheck(request.headers, objectKey, bucketName);
@@ -145,6 +151,7 @@ function objectGet(authInfo, request, returnTagCount, log, callback) {
145151
request.serverAccessLog.objectSize = objLength;
146152
}
147153
let byteRange;
154+
let partNumber = null;
148155
const streamingParams = {};
149156
if (request.headers.range) {
150157
const { range, error } = parseRange(request.headers.range,
@@ -211,8 +218,6 @@ function objectGet(authInfo, request, returnTagCount, log, callback) {
211218
if (dataLocator[0] && dataLocator[0].dataStoreType === 'azure') {
212219
dataLocator[0].azureStreamingOptions = streamingParams;
213220
}
214-
215-
let partNumber = null;
216221
if (request.query && request.query.partNumber !== undefined) {
217222
if (byteRange) {
218223
const error = errorInstances.InvalidRequest
@@ -301,6 +306,16 @@ function objectGet(authInfo, request, returnTagCount, log, callback) {
301306
dataLocator = setPartRanges(dataLocator, byteRange);
302307
}
303308
}
309+
// Checksum is not returned for partNumber requests because per-part
310+
// checksums are not yet stored (S3C-11073).
311+
if (checksumMode === 'ENABLED' && !byteRange && !partNumber) {
312+
const checksum = objMD.checksum;
313+
if (checksum) {
314+
responseMetaHeaders[`x-amz-checksum-${checksum.checksumAlgorithm}`]
315+
= checksum.checksumValue;
316+
responseMetaHeaders['x-amz-checksum-type'] = checksum.checksumType;
317+
}
318+
}
304319
// Check KMS Key access and usability before checking data
305320
// diff with AWS: for empty object (no dataLocator) KMS not checked
306321
return async.each(dataLocator || [],

‎lib/api/objectHead.js‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,10 @@ function objectHead(authInfo, request, log, callback) {
103103
const responseHeaders = collectResponseHeaders(objMD, corsHeaders,
104104
verCfg);
105105

106-
if (checksumMode === 'ENABLED') {
106+
const partNumber = getPartNumber(request.query);
107+
// Checksum is not returned for partNumber requests because per-part
108+
// checksums are not yet stored (S3C-11073).
109+
if (checksumMode === 'ENABLED' && !partNumber) {
107110
const checksum = objMD.checksum;
108111
if (checksum) {
109112
responseHeaders[`x-amz-checksum-${checksum.checksumAlgorithm}`]
@@ -150,7 +153,6 @@ function objectHead(authInfo, request, log, callback) {
150153
`bytes ${range[0]}-${range[1]}/${objLength}`;
151154
}
152155
}
153-
const partNumber = getPartNumber(request.query);
154156
if (partNumber !== undefined) {
155157
if (byteRange) {
156158
const error = errorInstances.InvalidRequest

‎tests/functional/aws-node-sdk/test/object/get.js‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,13 @@ const {
2222
const withV4 = require('../support/withV4');
2323
const BucketUtility = require('../../lib/utility/bucket-util');
2424
const changeObjectLock = require('../../../../utilities/objectLock-util');
25+
const { algorithms } = require('../../../../../lib/api/apiUtils/integrity/validateChecksums');
26+
27+
const { crc64NvmeCrtContainer } = require('@aws-sdk/middleware-flexible-checksums');
28+
if (crc64NvmeCrtContainer) {
29+
const { CrtCrc64Nvme } = require('@aws-sdk/crc64-nvme-crt');
30+
crc64NvmeCrtContainer.CrtCrc64Nvme = CrtCrc64Nvme;
31+
}
2532

2633
const changeLockPromise = promisify(changeObjectLock);
2734

@@ -1121,3 +1128,98 @@ describeSkipIfCeph('GET object with object lock', () => {
11211128
});
11221129
});
11231130
});
1131+
1132+
describe('GET object checksum mode', () => {
1133+
withV4(sigCfg => {
1134+
let bucketUtil;
1135+
let s3;
1136+
const checksumBucket = 'checksum-getobject-test';
1137+
const checksumKey = 'checksum-test-object';
1138+
const body = Buffer.from('checksum test body');
1139+
1140+
const expectedDigests = {};
1141+
1142+
const checksumAlgorithms = [
1143+
{ algorithm: 'SHA256', responseField: 'ChecksumSHA256', internalName: 'sha256' },
1144+
{ algorithm: 'SHA1', responseField: 'ChecksumSHA1', internalName: 'sha1' },
1145+
{ algorithm: 'CRC32', responseField: 'ChecksumCRC32', internalName: 'crc32' },
1146+
{ algorithm: 'CRC32C', responseField: 'ChecksumCRC32C', internalName: 'crc32c' },
1147+
{ algorithm: 'CRC64NVME', responseField: 'ChecksumCRC64NVME', internalName: 'crc64nvme' },
1148+
];
1149+
1150+
before(async () => {
1151+
// Disable automatic response checksum validation so the SDK does
1152+
// not silently add x-amz-checksum-mode: ENABLED to every GetObject
1153+
// request, which would interfere with the "mode not set" test.
1154+
bucketUtil = new BucketUtility('default',
1155+
{ ...sigCfg, responseChecksumValidation: 'WHEN_REQUIRED' });
1156+
s3 = bucketUtil.s3;
1157+
await s3.send(new CreateBucketCommand({ Bucket: checksumBucket }));
1158+
1159+
for (const { internalName } of checksumAlgorithms) {
1160+
expectedDigests[internalName] =
1161+
await algorithms[internalName].digest(body);
1162+
}
1163+
});
1164+
1165+
after(async () => {
1166+
await bucketUtil.empty(checksumBucket);
1167+
await s3.send(new DeleteBucketCommand({ Bucket: checksumBucket }));
1168+
});
1169+
1170+
checksumAlgorithms.forEach(({ algorithm, responseField, internalName }) => {
1171+
it(`should return ${responseField} and ChecksumType when ChecksumMode is ENABLED`, async () => {
1172+
const putRes = await s3.send(new PutObjectCommand({
1173+
Bucket: checksumBucket,
1174+
Key: checksumKey,
1175+
Body: body,
1176+
ChecksumAlgorithm: algorithm,
1177+
}));
1178+
const storedChecksum = putRes[responseField];
1179+
assert(storedChecksum, `Expected ${responseField} in PutObject response`);
1180+
1181+
const getRes = await s3.send(new GetObjectCommand({
1182+
Bucket: checksumBucket,
1183+
Key: checksumKey,
1184+
ChecksumMode: 'ENABLED',
1185+
}));
1186+
assert.strictEqual(getRes[responseField], expectedDigests[internalName],
1187+
`${responseField} value mismatch`);
1188+
assert.strictEqual(getRes[responseField], storedChecksum);
1189+
assert.strictEqual(getRes.ChecksumType, 'FULL_OBJECT');
1190+
});
1191+
});
1192+
1193+
it('should not return checksum headers when ChecksumMode is not set', async () => {
1194+
await s3.send(new PutObjectCommand({
1195+
Bucket: checksumBucket,
1196+
Key: checksumKey,
1197+
Body: body,
1198+
ChecksumAlgorithm: 'SHA256',
1199+
}));
1200+
1201+
const getRes = await s3.send(new GetObjectCommand({
1202+
Bucket: checksumBucket,
1203+
Key: checksumKey,
1204+
}));
1205+
assert.strictEqual(getRes.ChecksumSHA256, undefined);
1206+
assert.strictEqual(getRes.ChecksumType, undefined);
1207+
});
1208+
1209+
it('should return an error when ChecksumMode is not ENABLED', async () => {
1210+
await s3.send(new PutObjectCommand({
1211+
Bucket: checksumBucket,
1212+
Key: checksumKey,
1213+
Body: body,
1214+
}));
1215+
1216+
await assert.rejects(
1217+
s3.send(new GetObjectCommand({
1218+
Bucket: checksumBucket,
1219+
Key: checksumKey,
1220+
ChecksumMode: 'DISABLED',
1221+
})),
1222+
);
1223+
});
1224+
});
1225+
});

‎tests/functional/aws-node-sdk/test/object/objectHead.js‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -556,6 +556,14 @@ describe('HEAD object checksum mode', () => {
556556
const checksumKey = 'checksum-test-object';
557557
const body = Buffer.from('checksum test body');
558558

559+
const checksumAlgorithms = [
560+
{ algorithm: 'SHA256', responseField: 'ChecksumSHA256', internalName: 'sha256' },
561+
{ algorithm: 'SHA1', responseField: 'ChecksumSHA1', internalName: 'sha1' },
562+
{ algorithm: 'CRC32', responseField: 'ChecksumCRC32', internalName: 'crc32' },
563+
{ algorithm: 'CRC32C', responseField: 'ChecksumCRC32C', internalName: 'crc32c' },
564+
{ algorithm: 'CRC64NVME', responseField: 'ChecksumCRC64NVME', internalName: 'crc64nvme' },
565+
];
566+
559567
// Expected base64-encoded digests of `body` for each algorithm,
560568
// computed once in the before hook (crc64nvme digest is async).
561569
const expectedDigests = {};
@@ -577,14 +585,6 @@ describe('HEAD object checksum mode', () => {
577585
await s3.send(new DeleteBucketCommand({ Bucket: checksumBucket }));
578586
});
579587

580-
const checksumAlgorithms = [
581-
{ algorithm: 'SHA256', responseField: 'ChecksumSHA256', internalName: 'sha256' },
582-
{ algorithm: 'SHA1', responseField: 'ChecksumSHA1', internalName: 'sha1' },
583-
{ algorithm: 'CRC32', responseField: 'ChecksumCRC32', internalName: 'crc32' },
584-
{ algorithm: 'CRC32C', responseField: 'ChecksumCRC32C', internalName: 'crc32c' },
585-
{ algorithm: 'CRC64NVME', responseField: 'ChecksumCRC64NVME', internalName: 'crc64nvme' },
586-
];
587-
588588
checksumAlgorithms.forEach(({ algorithm, responseField, internalName }) => {
589589
it(`should return ${responseField} and ChecksumType when ChecksumMode is ENABLED`, async () => {
590590
const putRes = await s3.send(new PutObjectCommand({

‎tests/unit/api/objectGet.js‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@ const assert = require('assert');
22
const async = require('async');
33
const crypto = require('crypto');
44
const { parseString } = require('xml2js');
5+
const { models } = require('arsenal');
6+
const { ObjectMD, ObjectMDChecksum } = models;
7+
const { algorithms } = require('../../../lib/api/apiUtils/integrity/validateChecksums');
58

69
const { bucketPut } = require('../../../lib/api/bucketPut');
710
const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers');
@@ -518,3 +521,155 @@ describe('objectGet API', () => {
518521
});
519522
});
520523
});
524+
525+
describe('objectGet API - x-amz-checksum-mode', () => {
526+
const checksumAlgorithms = [
527+
{ name: 'sha256', header: 'x-amz-checksum-sha256' },
528+
{ name: 'sha1', header: 'x-amz-checksum-sha1' },
529+
{ name: 'crc32', header: 'x-amz-checksum-crc32' },
530+
{ name: 'crc32c', header: 'x-amz-checksum-crc32c' },
531+
{ name: 'crc64nvme', header: 'x-amz-checksum-crc64nvme' },
532+
];
533+
534+
const expectedDigests = {};
535+
536+
before(done => {
537+
Promise.all(checksumAlgorithms.map(async ({ name }) => {
538+
expectedDigests[name] = await algorithms[name].digest(postBody);
539+
})).then(() => done(), done);
540+
});
541+
542+
beforeEach(() => cleanup());
543+
544+
checksumAlgorithms.forEach(({ name, header }) => {
545+
it(`should return ${header} and x-amz-checksum-type when mode is ENABLED`, done => {
546+
const md = new ObjectMD(mdColdHelper.baseMd)
547+
.setChecksum(new ObjectMDChecksum(name, expectedDigests[name], 'FULL_OBJECT'));
548+
mdColdHelper.putBucketMock(bucketName, null, () =>
549+
mdColdHelper.putObjectMock(bucketName, objectName, md, () => {
550+
const req = {
551+
bucketName,
552+
namespace,
553+
objectKey: objectName,
554+
headers: { 'x-amz-checksum-mode': 'ENABLED' },
555+
url: `/${bucketName}/${objectName}`,
556+
query: {},
557+
};
558+
objectGet(authInfo, req, false, log, (err, _locator, headers) => {
559+
assert.ifError(err);
560+
assert.strictEqual(headers[header], expectedDigests[name]);
561+
assert.strictEqual(headers['x-amz-checksum-type'], 'FULL_OBJECT');
562+
done();
563+
});
564+
}));
565+
});
566+
});
567+
568+
it('should not return checksum headers when mode is ENABLED but object has no checksum', done => {
569+
mdColdHelper.putBucketMock(bucketName, null, () =>
570+
mdColdHelper.putObjectMock(bucketName, objectName, undefined, () => {
571+
const req = {
572+
bucketName,
573+
namespace,
574+
objectKey: objectName,
575+
headers: { 'x-amz-checksum-mode': 'ENABLED' },
576+
url: `/${bucketName}/${objectName}`,
577+
query: {},
578+
};
579+
objectGet(authInfo, req, false, log, (err, _locator, headers) => {
580+
assert.ifError(err);
581+
checksumAlgorithms.forEach(({ header }) =>
582+
assert.strictEqual(headers[header], undefined));
583+
assert.strictEqual(headers['x-amz-checksum-type'], undefined);
584+
done();
585+
});
586+
}));
587+
});
588+
589+
it('should not return checksum headers when x-amz-checksum-mode is not set', done => {
590+
const md = new ObjectMD(mdColdHelper.baseMd)
591+
.setChecksum(new ObjectMDChecksum('sha256', expectedDigests.sha256, 'FULL_OBJECT'));
592+
mdColdHelper.putBucketMock(bucketName, null, () =>
593+
mdColdHelper.putObjectMock(bucketName, objectName, md, () => {
594+
const req = {
595+
bucketName,
596+
namespace,
597+
objectKey: objectName,
598+
headers: {},
599+
url: `/${bucketName}/${objectName}`,
600+
query: {},
601+
};
602+
objectGet(authInfo, req, false, log, (err, _locator, headers) => {
603+
assert.ifError(err);
604+
checksumAlgorithms.forEach(({ header }) =>
605+
assert.strictEqual(headers[header], undefined));
606+
assert.strictEqual(headers['x-amz-checksum-type'], undefined);
607+
done();
608+
});
609+
}));
610+
});
611+
612+
it('should return InvalidArgument when x-amz-checksum-mode is not ENABLED', done => {
613+
const req = {
614+
bucketName,
615+
namespace,
616+
objectKey: objectName,
617+
headers: { 'x-amz-checksum-mode': 'DISABLED' },
618+
url: `/${bucketName}/${objectName}`,
619+
query: {},
620+
};
621+
objectGet(authInfo, req, false, log, err => {
622+
assert.strictEqual(err.is.InvalidArgument, true);
623+
done();
624+
});
625+
});
626+
627+
it('should not return checksum headers when Range header is set', done => {
628+
const md = new ObjectMD(mdColdHelper.baseMd)
629+
.setChecksum(new ObjectMDChecksum('sha256', expectedDigests.sha256, 'FULL_OBJECT'));
630+
mdColdHelper.putBucketMock(bucketName, null, () =>
631+
mdColdHelper.putObjectMock(bucketName, objectName, md, () => {
632+
const req = {
633+
bucketName,
634+
namespace,
635+
objectKey: objectName,
636+
headers: {
637+
'x-amz-checksum-mode': 'ENABLED',
638+
range: 'bytes=0-3',
639+
},
640+
url: `/${bucketName}/${objectName}`,
641+
query: {},
642+
};
643+
objectGet(authInfo, req, false, log, (err, _locator, headers) => {
644+
assert.ifError(err);
645+
checksumAlgorithms.forEach(({ header }) =>
646+
assert.strictEqual(headers[header], undefined));
647+
assert.strictEqual(headers['x-amz-checksum-type'], undefined);
648+
done();
649+
});
650+
}));
651+
});
652+
653+
it('should not return checksum headers when partNumber is set', done => {
654+
const md = new ObjectMD(mdColdHelper.baseMd)
655+
.setChecksum(new ObjectMDChecksum('sha256', expectedDigests.sha256, 'FULL_OBJECT'));
656+
mdColdHelper.putBucketMock(bucketName, null, () =>
657+
mdColdHelper.putObjectMock(bucketName, objectName, md, () => {
658+
const req = {
659+
bucketName,
660+
namespace,
661+
objectKey: objectName,
662+
headers: { 'x-amz-checksum-mode': 'ENABLED' },
663+
url: `/${bucketName}/${objectName}`,
664+
query: { partNumber: '1' },
665+
};
666+
objectGet(authInfo, req, false, log, (err, _locator, headers) => {
667+
assert.ifError(err);
668+
checksumAlgorithms.forEach(({ header }) =>
669+
assert.strictEqual(headers[header], undefined));
670+
assert.strictEqual(headers['x-amz-checksum-type'], undefined);
671+
done();
672+
});
673+
}));
674+
});
675+
});

0 commit comments

Comments
 (0)