Skip to content

Commit e940463

Browse files
bert-ebert-e
committed
Merge branch 'improvement/CLDSRV-873-head-get-object-return-checksum' into q/9.4
2 parents 38c7ff0 + 202d8e0 commit e940463

File tree

7 files changed

+525
-3
lines changed

7 files changed

+525
-3
lines changed

‎lib/api/objectGet.js‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,12 @@ function objectGet(authInfo, request, returnTagCount, log, callback) {
4141
const bucketName = request.bucketName;
4242
const objectKey = request.objectKey;
4343

44+
const checksumMode = request.headers['x-amz-checksum-mode'];
45+
if (checksumMode !== undefined && checksumMode !== 'ENABLED') {
46+
log.debug('invalid x-amz-checksum-mode', { checksumMode });
47+
return callback(errors.InvalidArgument);
48+
}
49+
4450
// returns name of location to get from and key if successful
4551
const locCheckResult =
4652
locationHeaderCheck(request.headers, objectKey, bucketName);
@@ -145,6 +151,7 @@ function objectGet(authInfo, request, returnTagCount, log, callback) {
145151
request.serverAccessLog.objectSize = objLength;
146152
}
147153
let byteRange;
154+
let partNumber = null;
148155
const streamingParams = {};
149156
if (request.headers.range) {
150157
const { range, error } = parseRange(request.headers.range,
@@ -211,8 +218,6 @@ function objectGet(authInfo, request, returnTagCount, log, callback) {
211218
if (dataLocator[0] && dataLocator[0].dataStoreType === 'azure') {
212219
dataLocator[0].azureStreamingOptions = streamingParams;
213220
}
214-
215-
let partNumber = null;
216221
if (request.query && request.query.partNumber !== undefined) {
217222
if (byteRange) {
218223
const error = errorInstances.InvalidRequest
@@ -301,6 +306,16 @@ function objectGet(authInfo, request, returnTagCount, log, callback) {
301306
dataLocator = setPartRanges(dataLocator, byteRange);
302307
}
303308
}
309+
// Checksum is not returned for partNumber requests because per-part
310+
// checksums are not yet stored (S3C-11073).
311+
if (checksumMode === 'ENABLED' && !byteRange && !partNumber) {
312+
const checksum = objMD.checksum;
313+
if (checksum) {
314+
responseMetaHeaders[`x-amz-checksum-${checksum.checksumAlgorithm}`]
315+
= checksum.checksumValue;
316+
responseMetaHeaders['x-amz-checksum-type'] = checksum.checksumType;
317+
}
318+
}
304319
// Check KMS Key access and usability before checking data
305320
// diff with AWS: for empty object (no dataLocator) KMS not checked
306321
return async.each(dataLocator || [],

‎lib/api/objectHead.js‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,12 @@ function objectHead(authInfo, request, log, callback) {
3232
const bucketName = request.bucketName;
3333
const objectKey = request.objectKey;
3434

35+
const checksumMode = request.headers['x-amz-checksum-mode'];
36+
if (checksumMode !== undefined && checksumMode !== 'ENABLED') {
37+
log.debug('invalid x-amz-checksum-mode', { checksumMode });
38+
return callback(errors.InvalidArgument);
39+
}
40+
3541
const decodedVidResult = decodeVersionId(request.query);
3642
if (decodedVidResult instanceof Error) {
3743
log.trace('invalid versionId query', {
@@ -97,6 +103,18 @@ function objectHead(authInfo, request, log, callback) {
97103
const responseHeaders = collectResponseHeaders(objMD, corsHeaders,
98104
verCfg);
99105

106+
const partNumber = getPartNumber(request.query);
107+
// Checksum is not returned for partNumber requests because per-part
108+
// checksums are not yet stored (S3C-11073).
109+
if (checksumMode === 'ENABLED' && !partNumber) {
110+
const checksum = objMD.checksum;
111+
if (checksum) {
112+
responseHeaders[`x-amz-checksum-${checksum.checksumAlgorithm}`]
113+
= checksum.checksumValue;
114+
responseHeaders['x-amz-checksum-type'] = checksum.checksumType;
115+
}
116+
}
117+
100118
setExpirationHeaders(responseHeaders, {
101119
lifecycleConfig: bucket.getLifecycleConfiguration(),
102120
objectParams: {
@@ -135,7 +153,6 @@ function objectHead(authInfo, request, log, callback) {
135153
`bytes ${range[0]}-${range[1]}/${objLength}`;
136154
}
137155
}
138-
const partNumber = getPartNumber(request.query);
139156
if (partNumber !== undefined) {
140157
if (byteRange) {
141158
const error = errorInstances.InvalidRequest

‎tests/functional/aws-node-sdk/test/object/get.js‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,13 @@ const {
2222
const withV4 = require('../support/withV4');
2323
const BucketUtility = require('../../lib/utility/bucket-util');
2424
const changeObjectLock = require('../../../../utilities/objectLock-util');
25+
const { algorithms } = require('../../../../../lib/api/apiUtils/integrity/validateChecksums');
26+
27+
const { crc64NvmeCrtContainer } = require('@aws-sdk/middleware-flexible-checksums');
28+
if (crc64NvmeCrtContainer) {
29+
const { CrtCrc64Nvme } = require('@aws-sdk/crc64-nvme-crt');
30+
crc64NvmeCrtContainer.CrtCrc64Nvme = CrtCrc64Nvme;
31+
}
2532

2633
const changeLockPromise = promisify(changeObjectLock);
2734

@@ -1121,3 +1128,102 @@ describeSkipIfCeph('GET object with object lock', () => {
11211128
});
11221129
});
11231130
});
1131+
1132+
describe('GET object checksum mode', () => {
1133+
withV4(sigCfg => {
1134+
let bucketUtil;
1135+
let s3;
1136+
const checksumBucket = 'checksum-getobject-test';
1137+
const checksumKey = 'checksum-test-object';
1138+
const body = Buffer.from('checksum test body');
1139+
1140+
const expectedDigests = {};
1141+
1142+
const checksumAlgorithms = [
1143+
{ algorithm: 'SHA256', responseField: 'ChecksumSHA256', internalName: 'sha256' },
1144+
{ algorithm: 'SHA1', responseField: 'ChecksumSHA1', internalName: 'sha1' },
1145+
{ algorithm: 'CRC32', responseField: 'ChecksumCRC32', internalName: 'crc32' },
1146+
{ algorithm: 'CRC32C', responseField: 'ChecksumCRC32C', internalName: 'crc32c' },
1147+
{ algorithm: 'CRC64NVME', responseField: 'ChecksumCRC64NVME', internalName: 'crc64nvme' },
1148+
];
1149+
1150+
before(async () => {
1151+
// Disable automatic response checksum validation so the SDK does
1152+
// not silently add x-amz-checksum-mode: ENABLED to every GetObject
1153+
// request, which would interfere with the "mode not set" test.
1154+
bucketUtil = new BucketUtility('default',
1155+
{ ...sigCfg, responseChecksumValidation: 'WHEN_REQUIRED' });
1156+
s3 = bucketUtil.s3;
1157+
await s3.send(new CreateBucketCommand({ Bucket: checksumBucket }));
1158+
1159+
for (const { internalName } of checksumAlgorithms) {
1160+
expectedDigests[internalName] =
1161+
await algorithms[internalName].digest(body);
1162+
}
1163+
});
1164+
1165+
after(async () => {
1166+
await bucketUtil.empty(checksumBucket);
1167+
await s3.send(new DeleteBucketCommand({ Bucket: checksumBucket }));
1168+
});
1169+
1170+
checksumAlgorithms.forEach(({ algorithm, responseField, internalName }) => {
1171+
it(`should return ${responseField} and ChecksumType when ChecksumMode is ENABLED`, async () => {
1172+
const putRes = await s3.send(new PutObjectCommand({
1173+
Bucket: checksumBucket,
1174+
Key: checksumKey,
1175+
Body: body,
1176+
ChecksumAlgorithm: algorithm,
1177+
}));
1178+
const storedChecksum = putRes[responseField];
1179+
assert(storedChecksum, `Expected ${responseField} in PutObject response`);
1180+
1181+
const getRes = await s3.send(new GetObjectCommand({
1182+
Bucket: checksumBucket,
1183+
Key: checksumKey,
1184+
ChecksumMode: 'ENABLED',
1185+
}));
1186+
assert.strictEqual(getRes[responseField], expectedDigests[internalName],
1187+
`${responseField} value mismatch`);
1188+
assert.strictEqual(getRes[responseField], storedChecksum);
1189+
assert.strictEqual(getRes.ChecksumType, 'FULL_OBJECT');
1190+
});
1191+
});
1192+
1193+
it('should not return checksum headers when ChecksumMode is not set', async () => {
1194+
await s3.send(new PutObjectCommand({
1195+
Bucket: checksumBucket,
1196+
Key: checksumKey,
1197+
Body: body,
1198+
ChecksumAlgorithm: 'SHA256',
1199+
}));
1200+
1201+
const getRes = await s3.send(new GetObjectCommand({
1202+
Bucket: checksumBucket,
1203+
Key: checksumKey,
1204+
}));
1205+
assert.strictEqual(getRes.ChecksumSHA256, undefined);
1206+
assert.strictEqual(getRes.ChecksumType, undefined);
1207+
});
1208+
1209+
it('should return an error when ChecksumMode is not ENABLED', async () => {
1210+
await s3.send(new PutObjectCommand({
1211+
Bucket: checksumBucket,
1212+
Key: checksumKey,
1213+
Body: body,
1214+
}));
1215+
1216+
await assert.rejects(
1217+
s3.send(new GetObjectCommand({
1218+
Bucket: checksumBucket,
1219+
Key: checksumKey,
1220+
ChecksumMode: 'DISABLED',
1221+
})),
1222+
err => {
1223+
assert.strictEqual(err.name, 'InvalidArgument');
1224+
return true;
1225+
},
1226+
);
1227+
});
1228+
});
1229+
});

‎tests/functional/aws-node-sdk/test/object/objectHead.js‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,17 @@ const {
1414
UploadPartCommand,
1515
CompleteMultipartUploadCommand,
1616
} = require('@aws-sdk/client-s3');
17+
// In older versions of @aws-sdk/middleware-flexible-checksums (<3.972), the
18+
// CRC64NVME implementation must be patched in manually from the CRT package.
19+
// Newer versions handle this internally via @aws-sdk/crc64-nvme, so the export
20+
// no longer exists and no registration is needed.
21+
const { crc64NvmeCrtContainer } = require('@aws-sdk/middleware-flexible-checksums');
22+
if (crc64NvmeCrtContainer) {
23+
const { CrtCrc64Nvme } = require('@aws-sdk/crc64-nvme-crt');
24+
crc64NvmeCrtContainer.CrtCrc64Nvme = CrtCrc64Nvme;
25+
}
1726

27+
const { algorithms } = require('../../../../../lib/api/apiUtils/integrity/validateChecksums');
1828
const changeObjectLock = require('../../../../utilities/objectLock-util');
1929
const withV4 = require('../support/withV4');
2030
const BucketUtility = require('../../lib/utility/bucket-util');
@@ -538,6 +548,100 @@ describe('HEAD object, conditions', () => {
538548
});
539549
});
540550

551+
describe('HEAD object checksum mode', () => {
552+
withV4(sigCfg => {
553+
let bucketUtil;
554+
let s3;
555+
const checksumBucket = 'checksum-headobject-test';
556+
const checksumKey = 'checksum-test-object';
557+
const body = Buffer.from('checksum test body');
558+
559+
const checksumAlgorithms = [
560+
{ algorithm: 'SHA256', responseField: 'ChecksumSHA256', internalName: 'sha256' },
561+
{ algorithm: 'SHA1', responseField: 'ChecksumSHA1', internalName: 'sha1' },
562+
{ algorithm: 'CRC32', responseField: 'ChecksumCRC32', internalName: 'crc32' },
563+
{ algorithm: 'CRC32C', responseField: 'ChecksumCRC32C', internalName: 'crc32c' },
564+
{ algorithm: 'CRC64NVME', responseField: 'ChecksumCRC64NVME', internalName: 'crc64nvme' },
565+
];
566+
567+
// Expected base64-encoded digests of `body` for each algorithm,
568+
// computed once in the before hook (crc64nvme digest is async).
569+
const expectedDigests = {};
570+
571+
before(async () => {
572+
bucketUtil = new BucketUtility('default', sigCfg);
573+
s3 = bucketUtil.s3;
574+
await s3.send(new CreateBucketCommand({ Bucket: checksumBucket }));
575+
576+
for (const { internalName } of checksumAlgorithms) {
577+
// algorithms[internalName].digest() returns a base64 string
578+
expectedDigests[internalName] =
579+
await algorithms[internalName].digest(body);
580+
}
581+
});
582+
583+
after(async () => {
584+
await bucketUtil.empty(checksumBucket);
585+
await s3.send(new DeleteBucketCommand({ Bucket: checksumBucket }));
586+
});
587+
588+
checksumAlgorithms.forEach(({ algorithm, responseField, internalName }) => {
589+
it(`should return ${responseField} and ChecksumType when ChecksumMode is ENABLED`, async () => {
590+
const putRes = await s3.send(new PutObjectCommand({
591+
Bucket: checksumBucket,
592+
Key: checksumKey,
593+
Body: body,
594+
ChecksumAlgorithm: algorithm,
595+
}));
596+
const storedChecksum = putRes[responseField];
597+
assert(storedChecksum, `Expected ${responseField} in PutObject response`);
598+
599+
const headRes = await s3.send(new HeadObjectCommand({
600+
Bucket: checksumBucket,
601+
Key: checksumKey,
602+
ChecksumMode: 'ENABLED',
603+
}));
604+
assert.strictEqual(headRes[responseField], expectedDigests[internalName],
605+
`${responseField} value mismatch`);
606+
assert.strictEqual(headRes[responseField], storedChecksum);
607+
assert.strictEqual(headRes.ChecksumType, 'FULL_OBJECT');
608+
});
609+
});
610+
611+
it('should not return checksum headers when ChecksumMode is not set', async () => {
612+
await s3.send(new PutObjectCommand({
613+
Bucket: checksumBucket,
614+
Key: checksumKey,
615+
Body: body,
616+
ChecksumAlgorithm: 'SHA256',
617+
}));
618+
619+
const headRes = await s3.send(new HeadObjectCommand({
620+
Bucket: checksumBucket,
621+
Key: checksumKey,
622+
}));
623+
assert.strictEqual(headRes.ChecksumSHA256, undefined);
624+
assert.strictEqual(headRes.ChecksumType, undefined);
625+
});
626+
627+
it('should return an error when ChecksumMode is not ENABLED', async () => {
628+
await s3.send(new PutObjectCommand({
629+
Bucket: checksumBucket,
630+
Key: checksumKey,
631+
Body: body,
632+
}));
633+
634+
await assert.rejects(
635+
s3.send(new HeadObjectCommand({
636+
Bucket: checksumBucket,
637+
Key: checksumKey,
638+
ChecksumMode: 'DISABLED',
639+
})),
640+
);
641+
});
642+
});
643+
});
644+
541645
const isCEPH = process.env.CI_CEPH !== undefined;
542646
const describeSkipIfCeph = isCEPH ? describe.skip : describe;
543647

0 commit comments

Comments
 (0)