Skip to content

Commit d396ffe

Browse files
leif-scalityleif-scality
committed
CLDSRV-874: return checksum in GetObjectAttributes
1 parent e940463 commit d396ffe

File tree

6 files changed

+384
-92
lines changed

6 files changed

+384
-92
lines changed

‎lib/api/apiUtils/integrity/validateChecksums.js‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ function uint32ToBase64(num) {
6969

7070
const algorithms = Object.freeze({
7171
crc64nvme: {
72+
getObjectAttributesXMLTag: 'ChecksumCRC64NVME',
7273
digest: async data => {
7374
const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
7475
const crc = new CrtCrc64Nvme();
@@ -84,6 +85,7 @@ const algorithms = Object.freeze({
8485
createHash: () => new CrtCrc64Nvme()
8586
},
8687
crc32: {
88+
getObjectAttributesXMLTag: 'ChecksumCRC32',
8789
digest: data => {
8890
const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
8991
return uint32ToBase64(new Crc32().update(input).digest() >>> 0); // >>> 0 coerce number to uint32
@@ -96,6 +98,7 @@ const algorithms = Object.freeze({
9698
createHash: () => new Crc32()
9799
},
98100
crc32c: {
101+
getObjectAttributesXMLTag: 'ChecksumCRC32C',
99102
digest: data => {
100103
const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
101104
return uint32ToBase64(new Crc32c().update(input).digest() >>> 0); // >>> 0 coerce number to uint32
@@ -105,6 +108,7 @@ const algorithms = Object.freeze({
105108
createHash: () => new Crc32c()
106109
},
107110
sha1: {
111+
getObjectAttributesXMLTag: 'ChecksumSHA1',
108112
digest: data => {
109113
const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
110114
return crypto.createHash('sha1').update(input).digest('base64');
@@ -114,6 +118,7 @@ const algorithms = Object.freeze({
114118
createHash: () => crypto.createHash('sha1')
115119
},
116120
sha256: {
121+
getObjectAttributesXMLTag: 'ChecksumSHA256',
117122
digest: data => {
118123
const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
119124
return crypto.createHash('sha256').update(input).digest('base64');

‎lib/api/apiUtils/object/objectAttributes.js‎

Lines changed: 67 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
const { errorInstances } = require('arsenal');
22
const { getPartCountFromMd5 } = require('./partInfo');
3+
const { algorithms } = require('../integrity/validateChecksums');
34

45
/**
56
* Parse and validate attribute headers from a request.
@@ -53,59 +54,80 @@ function parseAttributesHeaders(headers, headerName, supportedAttributes) {
5354
* @param {Object.<string, any>} userMetadata - Key-value pairs of user-defined metadata.
5455
* @param {string[]} requestedAttrs - A list of specific attributes to include in the output.
5556
* Supports 'ETag', 'ObjectParts', 'StorageClass', 'ObjectSize',
56-
* 'RestoreStatus', and 'x-amz-meta-*' for all user metadata.
57+
* 'Checksum', 'RestoreStatus', and 'x-amz-meta-*' for all user metadata.
5758
* @param {string[]} xml - The string array acting as the output buffer/collector.
59+
* @param {object} log - Werelogs logger.
5860
* @returns {void} - this function does not return a value, it mutates the `xml` param.
5961
*/
60-
function buildAttributesXml(objectMD, userMetadata, requestedAttrs, xml) {
61-
const customAttributes = new Set();
62-
for (const attribute of requestedAttrs) {
63-
switch (attribute) {
64-
case 'ETag':
65-
xml.push(`<ETag>${objectMD['content-md5']}</ETag>`);
66-
break;
67-
case 'ObjectParts': {
68-
const partCount = getPartCountFromMd5(objectMD);
69-
if (partCount) {
70-
xml.push(
71-
'<ObjectParts>',
72-
`<PartsCount>${partCount}</PartsCount>`,
73-
'</ObjectParts>',
74-
);
75-
}
76-
break;
77-
}
78-
case 'StorageClass':
79-
xml.push(`<StorageClass>${objectMD['x-amz-storage-class']}</StorageClass>`);
80-
break;
81-
case 'ObjectSize':
82-
xml.push(`<ObjectSize>${objectMD['content-length']}</ObjectSize>`);
83-
break;
84-
case 'RestoreStatus':
85-
xml.push('<RestoreStatus>');
86-
xml.push(`<IsRestoreInProgress>${!!objectMD.restoreStatus?.inProgress}</IsRestoreInProgress>`);
62+
function buildAttributesXml(objectMD, userMetadata, requestedAttrs, xml, log) {
63+
const customAttributes = new Set();
64+
for (const attribute of requestedAttrs) {
65+
switch (attribute) {
66+
case 'ETag':
67+
xml.push(`<ETag>${objectMD['content-md5']}</ETag>`);
68+
break;
69+
case 'ObjectParts': {
70+
const partCount = getPartCountFromMd5(objectMD);
71+
if (partCount) {
72+
xml.push(
73+
'<ObjectParts>',
74+
`<PartsCount>${partCount}</PartsCount>`,
75+
'</ObjectParts>',
76+
);
77+
}
78+
break;
79+
}
80+
case 'StorageClass':
81+
xml.push(`<StorageClass>${objectMD['x-amz-storage-class']}</StorageClass>`);
82+
break;
83+
case 'ObjectSize':
84+
xml.push(`<ObjectSize>${objectMD['content-length']}</ObjectSize>`);
85+
break;
86+
case 'Checksum': {
87+
const { checksum } = objectMD;
88+
if (checksum) {
89+
const algo = algorithms[checksum.checksumAlgorithm];
90+
if (!algo) {
91+
log.error('unknown checksum algorithm in object metadata', {
92+
checksumAlgorithm: checksum.checksumAlgorithm,
93+
});
94+
break;
95+
}
96+
const tag = algo.getObjectAttributesXMLTag;
97+
xml.push(
98+
'<Checksum>',
99+
`<${tag}>${checksum.checksumValue}</${tag}>`,
100+
`<ChecksumType>${checksum.checksumType}</ChecksumType>`,
101+
'</Checksum>',
102+
);
103+
}
104+
break;
105+
}
106+
case 'RestoreStatus':
107+
xml.push('<RestoreStatus>');
108+
xml.push(`<IsRestoreInProgress>${!!objectMD.restoreStatus?.inProgress}</IsRestoreInProgress>`);
87109

88-
if (objectMD.restoreStatus?.expiryDate) {
89-
xml.push(`<RestoreExpiryDate>${objectMD.restoreStatus?.expiryDate}</RestoreExpiryDate>`);
90-
}
110+
if (objectMD.restoreStatus?.expiryDate) {
111+
xml.push(`<RestoreExpiryDate>${objectMD.restoreStatus?.expiryDate}</RestoreExpiryDate>`);
112+
}
91113

92-
xml.push('</RestoreStatus>');
93-
break;
94-
case 'x-amz-meta-*':
95-
for (const key of Object.keys(userMetadata)) {
96-
customAttributes.add(key);
97-
}
98-
break;
99-
default:
100-
if (userMetadata[attribute]) {
101-
customAttributes.add(attribute);
114+
xml.push('</RestoreStatus>');
115+
break;
116+
case 'x-amz-meta-*':
117+
for (const key of Object.keys(userMetadata)) {
118+
customAttributes.add(key);
119+
}
120+
break;
121+
default:
122+
if (userMetadata[attribute]) {
123+
customAttributes.add(attribute);
124+
}
102125
}
103126
}
104-
}
105127

106-
for (const key of customAttributes) {
107-
xml.push(`<${key}>${userMetadata[key]}</${key}>`);
108-
}
128+
for (const key of customAttributes) {
129+
xml.push(`<${key}>${userMetadata[key]}</${key}>`);
130+
}
109131
}
110132

111133
module.exports = {

‎lib/api/objectGetAttributes.js‎

Lines changed: 4 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -17,16 +17,17 @@ const OBJECT_GET_ATTRIBUTES = 'objectGetAttributes';
1717
* buildXmlResponse - Build XML response for GetObjectAttributes
1818
* @param {object} objMD - object metadata
1919
* @param {Set<string>} requestedAttrs - set of requested attribute names
20+
* @param {object} log - Werelogs logger
2021
* @returns {string} XML response
2122
*/
22-
function buildXmlResponse(objMD, requestedAttrs) {
23+
function buildXmlResponse(objMD, requestedAttrs, log) {
2324
const xml = [
2425
'<?xml version="1.0" encoding="UTF-8"?>',
2526
'<GetObjectAttributesResponse>',
2627
];
2728

2829
const userMetadata = getUserMetadata(objMD);
29-
buildAttributesXml(objMD, userMetadata, requestedAttrs, xml);
30+
buildAttributesXml(objMD, userMetadata, requestedAttrs, xml, log);
3031

3132
xml.push('</GetObjectAttributesResponse>');
3233
return xml.join('');
@@ -150,18 +151,6 @@ async function objectGetAttributes(authInfo, request, log, callback) {
150151
true,
151152
);
152153

153-
if (requestedAttrs.has('Checksum')) {
154-
log.debug('Checksum attribute requested but not implemented', {
155-
method: OBJECT_GET_ATTRIBUTES,
156-
bucket: bucketName,
157-
key: objectKey,
158-
versionId,
159-
});
160-
const err = errors.NotImplemented.customizeDescription('Checksum attribute is not implemented');
161-
err.responseHeaders = responseHeaders;
162-
throw err;
163-
}
164-
165154
pushMetric(OBJECT_GET_ATTRIBUTES, log, {
166155
authInfo,
167156
bucket: bucketName,
@@ -170,7 +159,7 @@ async function objectGetAttributes(authInfo, request, log, callback) {
170159
location: objectMD?.dataStoreName,
171160
});
172161

173-
const xml = buildXmlResponse(objectMD, requestedAttrs);
162+
const xml = buildXmlResponse(objectMD, requestedAttrs, log);
174163
return { xml, responseHeaders };
175164
}
176165

‎tests/functional/aws-node-sdk/test/object/objectGetAttributes.js‎

Lines changed: 116 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ const {
1212
const { GetObjectAttributesExtendedCommand } = require('@scality/cloudserverclient');
1313
const withV4 = require('../support/withV4');
1414
const BucketUtility = require('../../lib/utility/bucket-util');
15+
const { algorithms } = require('../../../../../lib/api/apiUtils/integrity/validateChecksums');
1516

1617
const bucket = 'testbucket';
1718
const key = 'testobject';
@@ -30,7 +31,9 @@ describe('objectGetAttributes', () => {
3031

3132
beforeEach(async () => {
3233
await s3.send(new CreateBucketCommand({ Bucket: bucket }));
33-
await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: body }));
34+
await s3.send(new PutObjectCommand({
35+
Bucket: bucket, Key: key, Body: body, ChecksumAlgorithm: 'CRC64NVME',
36+
}));
3437
});
3538

3639
afterEach(async () => {
@@ -119,18 +122,28 @@ describe('objectGetAttributes', () => {
119122
assert.strictEqual(data.ETag, expectedMD5);
120123
});
121124

122-
it('should fail with NotImplemented when Checksum is requested', async () => {
123-
try {
124-
await s3.send(new GetObjectAttributesCommand({
125-
Bucket: bucket,
126-
Key: key,
127-
ObjectAttributes: ['Checksum'],
128-
}));
129-
assert.fail('Expected NotImplemented error');
130-
} catch (err) {
131-
assert.strictEqual(err.name, 'NotImplemented');
132-
assert.strictEqual(err.message, 'Checksum attribute is not implemented');
133-
}
125+
it('should return ChecksumCRC64NVME for object', async () => {
126+
const data = await s3.send(new GetObjectAttributesCommand({
127+
Bucket: bucket,
128+
Key: key,
129+
ObjectAttributes: ['Checksum'],
130+
}));
131+
132+
assert(data.Checksum, 'Checksum should be present');
133+
assert(data.Checksum.ChecksumCRC64NVME, 'ChecksumCRC64NVME should be present');
134+
assert.strictEqual(data.Checksum.ChecksumType, 'FULL_OBJECT');
135+
});
136+
137+
it('should not return Checksum when not requested', async () => {
138+
const data = await s3.send(new GetObjectAttributesCommand({
139+
Bucket: bucket,
140+
Key: key,
141+
ObjectAttributes: ['ETag', 'ObjectSize'],
142+
}));
143+
144+
assert(data.ETag, 'ETag should be present');
145+
assert(data.ObjectSize, 'ObjectSize should be present');
146+
assert.strictEqual(data.Checksum, undefined, 'Checksum should not be present');
134147
});
135148

136149
it("shouldn't return ObjectParts for non-MPU objects", async () => {
@@ -480,3 +493,93 @@ describe('objectGetAttributes with user metadata', () => {
480493
});
481494
});
482495
});
496+
497+
describe('objectGetAttributes with checksum', () => {
498+
withV4(sigCfg => {
499+
let bucketUtil;
500+
let s3;
501+
const checksumBucket = 'checksum-getattr-test';
502+
const checksumKey = 'checksum-test-object';
503+
const checksumBody = Buffer.from('checksum test body');
504+
505+
const expectedDigests = {};
506+
507+
before(async () => {
508+
bucketUtil = new BucketUtility('default', sigCfg);
509+
s3 = bucketUtil.s3;
510+
await s3.send(new CreateBucketCommand({ Bucket: checksumBucket }));
511+
512+
for (const [name, algo] of Object.entries(algorithms)) {
513+
expectedDigests[name] = await algo.digest(checksumBody);
514+
}
515+
});
516+
517+
after(async () => {
518+
await bucketUtil.empty(checksumBucket);
519+
await s3.send(new DeleteBucketCommand({ Bucket: checksumBucket }));
520+
});
521+
522+
Object.entries(algorithms).forEach(([name, { getObjectAttributesXMLTag }]) => {
523+
const sdkAlgorithm = name.toUpperCase();
524+
525+
it(`should return ${getObjectAttributesXMLTag} when object has ${name} checksum`, async () => {
526+
await s3.send(new PutObjectCommand({
527+
Bucket: checksumBucket,
528+
Key: checksumKey,
529+
Body: checksumBody,
530+
ChecksumAlgorithm: sdkAlgorithm,
531+
}));
532+
533+
const data = await s3.send(new GetObjectAttributesCommand({
534+
Bucket: checksumBucket,
535+
Key: checksumKey,
536+
ObjectAttributes: ['Checksum'],
537+
}));
538+
539+
assert(data.Checksum, 'Checksum should be present');
540+
assert.strictEqual(data.Checksum[getObjectAttributesXMLTag], expectedDigests[name]);
541+
assert.strictEqual(data.Checksum.ChecksumType, 'FULL_OBJECT');
542+
});
543+
544+
it(`should return ${getObjectAttributesXMLTag} along with other attributes`, async () => {
545+
await s3.send(new PutObjectCommand({
546+
Bucket: checksumBucket,
547+
Key: checksumKey,
548+
Body: checksumBody,
549+
ChecksumAlgorithm: sdkAlgorithm,
550+
}));
551+
552+
const data = await s3.send(new GetObjectAttributesCommand({
553+
Bucket: checksumBucket,
554+
Key: checksumKey,
555+
ObjectAttributes: ['ETag', 'Checksum', 'ObjectSize'],
556+
}));
557+
558+
assert(data.ETag, 'ETag should be present');
559+
assert(data.ObjectSize, 'ObjectSize should be present');
560+
assert(data.Checksum, 'Checksum should be present');
561+
assert.strictEqual(data.Checksum[getObjectAttributesXMLTag], expectedDigests[name]);
562+
assert.strictEqual(data.Checksum.ChecksumType, 'FULL_OBJECT');
563+
});
564+
});
565+
566+
it('should not return Checksum when not requested', async () => {
567+
await s3.send(new PutObjectCommand({
568+
Bucket: checksumBucket,
569+
Key: checksumKey,
570+
Body: checksumBody,
571+
ChecksumAlgorithm: 'CRC64NVME',
572+
}));
573+
574+
const data = await s3.send(new GetObjectAttributesCommand({
575+
Bucket: checksumBucket,
576+
Key: checksumKey,
577+
ObjectAttributes: ['ETag', 'ObjectSize'],
578+
}));
579+
580+
assert(data.ETag, 'ETag should be present');
581+
assert(data.ObjectSize, 'ObjectSize should be present');
582+
assert.strictEqual(data.Checksum, undefined, 'Checksum should not be present');
583+
});
584+
});
585+
});

0 commit comments

Comments
 (0)