Skip to content

Commit 63c5751

Browse files
committed
CLDSRV-874: return checksum in GetObjectAttributes
1 parent e940463 commit 63c5751

File tree

6 files changed

+362
-68
lines changed

6 files changed

+362
-68
lines changed

lib/api/apiUtils/integrity/validateChecksums.js

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ function uint32ToBase64(num) {
6969

7070
const algorithms = Object.freeze({
7171
crc64nvme: {
72+
getObjectAttributesXMLTag: 'ChecksumCRC64NVME',
7273
digest: async data => {
7374
const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
7475
const crc = new CrtCrc64Nvme();
@@ -84,6 +85,7 @@ const algorithms = Object.freeze({
8485
createHash: () => new CrtCrc64Nvme()
8586
},
8687
crc32: {
88+
getObjectAttributesXMLTag: 'ChecksumCRC32',
8789
digest: data => {
8890
const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
8991
return uint32ToBase64(new Crc32().update(input).digest() >>> 0); // >>> 0 coerce number to uint32
@@ -96,6 +98,7 @@ const algorithms = Object.freeze({
9698
createHash: () => new Crc32()
9799
},
98100
crc32c: {
101+
getObjectAttributesXMLTag: 'ChecksumCRC32C',
99102
digest: data => {
100103
const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
101104
return uint32ToBase64(new Crc32c().update(input).digest() >>> 0); // >>> 0 coerce number to uint32
@@ -105,6 +108,7 @@ const algorithms = Object.freeze({
105108
createHash: () => new Crc32c()
106109
},
107110
sha1: {
111+
getObjectAttributesXMLTag: 'ChecksumSHA1',
108112
digest: data => {
109113
const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
110114
return crypto.createHash('sha1').update(input).digest('base64');
@@ -114,6 +118,7 @@ const algorithms = Object.freeze({
114118
createHash: () => crypto.createHash('sha1')
115119
},
116120
sha256: {
121+
getObjectAttributesXMLTag: 'ChecksumSHA256',
117122
digest: data => {
118123
const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
119124
return crypto.createHash('sha256').update(input).digest('base64');

lib/api/apiUtils/object/objectAttributes.js

Lines changed: 42 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
const { errorInstances } = require('arsenal');
22
const { getPartCountFromMd5 } = require('./partInfo');
3+
const { algorithms } = require('../integrity/validateChecksums');
34

45
/**
56
* Parse and validate attribute headers from a request.
@@ -17,28 +18,28 @@ const { getPartCountFromMd5 } = require('./partInfo');
1718
* ['ETag', 'ObjectSize', 'x-amz-meta-custom']
1819
*/
1920
function parseAttributesHeaders(headers, headerName, supportedAttributes) {
20-
const result = new Set();
21+
const result = new Set();
2122

22-
const rawValue = headers[headerName];
23-
if (rawValue === null || rawValue === undefined) {
24-
return result;
25-
}
26-
27-
for (const rawAttr of rawValue.split(',')) {
28-
let attr = rawAttr.trim();
23+
const rawValue = headers[headerName];
24+
if (rawValue === null || rawValue === undefined) {
25+
return result;
26+
}
2927

30-
if (!supportedAttributes.has(attr)) {
31-
attr = attr.toLowerCase();
32-
}
28+
for (const rawAttr of rawValue.split(',')) {
29+
let attr = rawAttr.trim();
3330

34-
if (!attr.startsWith('x-amz-meta-') && !supportedAttributes.has(attr)) {
35-
throw errorInstances.InvalidArgument.customizeDescription('Invalid attribute name specified.');
36-
}
31+
if (!supportedAttributes.has(attr)) {
32+
attr = attr.toLowerCase();
33+
}
3734

38-
result.add(attr);
35+
if (!attr.startsWith('x-amz-meta-') && !supportedAttributes.has(attr)) {
36+
throw errorInstances.InvalidArgument.customizeDescription('Invalid attribute name specified.');
3937
}
4038

41-
return result;
39+
result.add(attr);
40+
}
41+
42+
return result;
4243
}
4344

4445
/**
@@ -53,11 +54,12 @@ function parseAttributesHeaders(headers, headerName, supportedAttributes) {
5354
* @param {Object.<string, any>} userMetadata - Key-value pairs of user-defined metadata.
5455
* @param {string[]} requestedAttrs - A list of specific attributes to include in the output.
5556
* Supports 'ETag', 'ObjectParts', 'StorageClass', 'ObjectSize',
56-
* 'RestoreStatus', and 'x-amz-meta-*' for all user metadata.
57+
* 'Checksum', 'RestoreStatus', and 'x-amz-meta-*' for all user metadata.
5758
* @param {string[]} xml - The string array acting as the output buffer/collector.
59+
* @param {object} log - Werelogs logger.
5860
* @returns {void} - this function does not return a value, it mutates the `xml` param.
5961
*/
60-
function buildAttributesXml(objectMD, userMetadata, requestedAttrs, xml) {
62+
function buildAttributesXml(objectMD, userMetadata, requestedAttrs, xml, log) {
6163
const customAttributes = new Set();
6264
for (const attribute of requestedAttrs) {
6365
switch (attribute) {
@@ -81,12 +83,31 @@ function buildAttributesXml(objectMD, userMetadata, requestedAttrs, xml) {
8183
case 'ObjectSize':
8284
xml.push(`<ObjectSize>${objectMD['content-length']}</ObjectSize>`);
8385
break;
86+
case 'Checksum': {
87+
const { checksum } = objectMD;
88+
if (checksum) {
89+
const algo = algorithms[checksum.checksumAlgorithm];
90+
if (!algo) {
91+
log.error('unknown checksum algorithm in object metadata', {
92+
checksumAlgorithm: checksum.checksumAlgorithm,
93+
});
94+
break;
95+
}
96+
xml.push(
97+
'<Checksum>',
98+
`<${algo.getObjectAttributesXMLTag}>${checksum.checksumValue}</${algo.getObjectAttributesXMLTag}>`,
99+
`<ChecksumType>${checksum.checksumType}</ChecksumType>`,
100+
'</Checksum>',
101+
);
102+
}
103+
break;
104+
}
84105
case 'RestoreStatus':
85106
xml.push('<RestoreStatus>');
86107
xml.push(`<IsRestoreInProgress>${!!objectMD.restoreStatus?.inProgress}</IsRestoreInProgress>`);
87108

88109
if (objectMD.restoreStatus?.expiryDate) {
89-
xml.push(`<RestoreExpiryDate>${objectMD.restoreStatus?.expiryDate}</RestoreExpiryDate>`);
110+
xml.push(`<RestoreExpiryDate>${objectMD.restoreStatus?.expiryDate}</RestoreExpiryDate>`);
90111
}
91112

92113
xml.push('</RestoreStatus>');
@@ -109,6 +130,6 @@ function buildAttributesXml(objectMD, userMetadata, requestedAttrs, xml) {
109130
}
110131

111132
module.exports = {
112-
parseAttributesHeaders,
113-
buildAttributesXml,
133+
parseAttributesHeaders,
134+
buildAttributesXml,
114135
};

lib/api/objectGetAttributes.js

Lines changed: 4 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -17,16 +17,17 @@ const OBJECT_GET_ATTRIBUTES = 'objectGetAttributes';
1717
* buildXmlResponse - Build XML response for GetObjectAttributes
1818
* @param {object} objMD - object metadata
1919
* @param {Set<string>} requestedAttrs - set of requested attribute names
20+
* @param {object} log - Werelogs logger
2021
* @returns {string} XML response
2122
*/
22-
function buildXmlResponse(objMD, requestedAttrs) {
23+
function buildXmlResponse(objMD, requestedAttrs, log) {
2324
const xml = [
2425
'<?xml version="1.0" encoding="UTF-8"?>',
2526
'<GetObjectAttributesResponse>',
2627
];
2728

2829
const userMetadata = getUserMetadata(objMD);
29-
buildAttributesXml(objMD, userMetadata, requestedAttrs, xml);
30+
buildAttributesXml(objMD, userMetadata, requestedAttrs, xml, log);
3031

3132
xml.push('</GetObjectAttributesResponse>');
3233
return xml.join('');
@@ -150,18 +151,6 @@ async function objectGetAttributes(authInfo, request, log, callback) {
150151
true,
151152
);
152153

153-
if (requestedAttrs.has('Checksum')) {
154-
log.debug('Checksum attribute requested but not implemented', {
155-
method: OBJECT_GET_ATTRIBUTES,
156-
bucket: bucketName,
157-
key: objectKey,
158-
versionId,
159-
});
160-
const err = errors.NotImplemented.customizeDescription('Checksum attribute is not implemented');
161-
err.responseHeaders = responseHeaders;
162-
throw err;
163-
}
164-
165154
pushMetric(OBJECT_GET_ATTRIBUTES, log, {
166155
authInfo,
167156
bucket: bucketName,
@@ -170,7 +159,7 @@ async function objectGetAttributes(authInfo, request, log, callback) {
170159
location: objectMD?.dataStoreName,
171160
});
172161

173-
const xml = buildXmlResponse(objectMD, requestedAttrs);
162+
const xml = buildXmlResponse(objectMD, requestedAttrs, log);
174163
return { xml, responseHeaders };
175164
}
176165

tests/functional/aws-node-sdk/test/object/objectGetAttributes.js

Lines changed: 116 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ const {
1212
const { GetObjectAttributesExtendedCommand } = require('@scality/cloudserverclient');
1313
const withV4 = require('../support/withV4');
1414
const BucketUtility = require('../../lib/utility/bucket-util');
15+
const { algorithms } = require('../../../../../lib/api/apiUtils/integrity/validateChecksums');
1516

1617
const bucket = 'testbucket';
1718
const key = 'testobject';
@@ -30,7 +31,9 @@ describe('objectGetAttributes', () => {
3031

3132
beforeEach(async () => {
3233
await s3.send(new CreateBucketCommand({ Bucket: bucket }));
33-
await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: body }));
34+
await s3.send(new PutObjectCommand({
35+
Bucket: bucket, Key: key, Body: body, ChecksumAlgorithm: 'CRC64NVME',
36+
}));
3437
});
3538

3639
afterEach(async () => {
@@ -119,18 +122,28 @@ describe('objectGetAttributes', () => {
119122
assert.strictEqual(data.ETag, expectedMD5);
120123
});
121124

122-
it('should fail with NotImplemented when Checksum is requested', async () => {
123-
try {
124-
await s3.send(new GetObjectAttributesCommand({
125-
Bucket: bucket,
126-
Key: key,
127-
ObjectAttributes: ['Checksum'],
128-
}));
129-
assert.fail('Expected NotImplemented error');
130-
} catch (err) {
131-
assert.strictEqual(err.name, 'NotImplemented');
132-
assert.strictEqual(err.message, 'Checksum attribute is not implemented');
133-
}
125+
it('should return ChecksumCRC64NVME for object', async () => {
126+
const data = await s3.send(new GetObjectAttributesCommand({
127+
Bucket: bucket,
128+
Key: key,
129+
ObjectAttributes: ['Checksum'],
130+
}));
131+
132+
assert(data.Checksum, 'Checksum should be present');
133+
assert(data.Checksum.ChecksumCRC64NVME, 'ChecksumCRC64NVME should be present');
134+
assert.strictEqual(data.Checksum.ChecksumType, 'FULL_OBJECT');
135+
});
136+
137+
it('should not return Checksum when not requested', async () => {
138+
const data = await s3.send(new GetObjectAttributesCommand({
139+
Bucket: bucket,
140+
Key: key,
141+
ObjectAttributes: ['ETag', 'ObjectSize'],
142+
}));
143+
144+
assert(data.ETag, 'ETag should be present');
145+
assert(data.ObjectSize, 'ObjectSize should be present');
146+
assert.strictEqual(data.Checksum, undefined, 'Checksum should not be present');
134147
});
135148

136149
it("shouldn't return ObjectParts for non-MPU objects", async () => {
@@ -480,3 +493,93 @@ describe('objectGetAttributes with user metadata', () => {
480493
});
481494
});
482495
});
496+
497+
describe('objectGetAttributes with checksum', () => {
498+
withV4(sigCfg => {
499+
let bucketUtil;
500+
let s3;
501+
const checksumBucket = 'checksum-getattr-test';
502+
const checksumKey = 'checksum-test-object';
503+
const checksumBody = Buffer.from('checksum test body');
504+
505+
const expectedDigests = {};
506+
507+
before(async () => {
508+
bucketUtil = new BucketUtility('default', sigCfg);
509+
s3 = bucketUtil.s3;
510+
await s3.send(new CreateBucketCommand({ Bucket: checksumBucket }));
511+
512+
for (const [name, algo] of Object.entries(algorithms)) {
513+
expectedDigests[name] = await algo.digest(checksumBody);
514+
}
515+
});
516+
517+
after(async () => {
518+
await bucketUtil.empty(checksumBucket);
519+
await s3.send(new DeleteBucketCommand({ Bucket: checksumBucket }));
520+
});
521+
522+
Object.entries(algorithms).forEach(([name, { getObjectAttributesXMLTag }]) => {
523+
const sdkAlgorithm = name.toUpperCase();
524+
525+
it(`should return ${getObjectAttributesXMLTag} when object has ${name} checksum`, async () => {
526+
await s3.send(new PutObjectCommand({
527+
Bucket: checksumBucket,
528+
Key: checksumKey,
529+
Body: checksumBody,
530+
ChecksumAlgorithm: sdkAlgorithm,
531+
}));
532+
533+
const data = await s3.send(new GetObjectAttributesCommand({
534+
Bucket: checksumBucket,
535+
Key: checksumKey,
536+
ObjectAttributes: ['Checksum'],
537+
}));
538+
539+
assert(data.Checksum, 'Checksum should be present');
540+
assert.strictEqual(data.Checksum[getObjectAttributesXMLTag], expectedDigests[name]);
541+
assert.strictEqual(data.Checksum.ChecksumType, 'FULL_OBJECT');
542+
});
543+
544+
it(`should return ${getObjectAttributesXMLTag} along with other attributes`, async () => {
545+
await s3.send(new PutObjectCommand({
546+
Bucket: checksumBucket,
547+
Key: checksumKey,
548+
Body: checksumBody,
549+
ChecksumAlgorithm: sdkAlgorithm,
550+
}));
551+
552+
const data = await s3.send(new GetObjectAttributesCommand({
553+
Bucket: checksumBucket,
554+
Key: checksumKey,
555+
ObjectAttributes: ['ETag', 'Checksum', 'ObjectSize'],
556+
}));
557+
558+
assert(data.ETag, 'ETag should be present');
559+
assert(data.ObjectSize, 'ObjectSize should be present');
560+
assert(data.Checksum, 'Checksum should be present');
561+
assert.strictEqual(data.Checksum[getObjectAttributesXMLTag], expectedDigests[name]);
562+
assert.strictEqual(data.Checksum.ChecksumType, 'FULL_OBJECT');
563+
});
564+
});
565+
566+
it('should not return Checksum when not requested', async () => {
567+
await s3.send(new PutObjectCommand({
568+
Bucket: checksumBucket,
569+
Key: checksumKey,
570+
Body: checksumBody,
571+
ChecksumAlgorithm: 'CRC64NVME',
572+
}));
573+
574+
const data = await s3.send(new GetObjectAttributesCommand({
575+
Bucket: checksumBucket,
576+
Key: checksumKey,
577+
ObjectAttributes: ['ETag', 'ObjectSize'],
578+
}));
579+
580+
assert(data.ETag, 'ETag should be present');
581+
assert(data.ObjectSize, 'ObjectSize should be present');
582+
assert.strictEqual(data.Checksum, undefined, 'Checksum should not be present');
583+
});
584+
});
585+
});

0 commit comments

Comments
 (0)