Skip to content

Commit d1176e1

Browse files
leif-scalityleif-scality
committed
CLDSRV-874: return checksum in GetObjectAttributes
1 parent e940463 commit d1176e1

6 files changed

Lines changed: 303 additions & 33 deletions

File tree

‎lib/api/apiUtils/integrity/validateChecksums.js‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ function uint32ToBase64(num) {
6969

7070
const algorithms = Object.freeze({
7171
crc64nvme: {
72+
getObjectAttributeXMLTag: 'ChecksumCRC64NVME',
7273
digest: async data => {
7374
const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
7475
const crc = new CrtCrc64Nvme();
@@ -84,6 +85,7 @@ const algorithms = Object.freeze({
8485
createHash: () => new CrtCrc64Nvme()
8586
},
8687
crc32: {
88+
getObjectAttributeXMLTag: 'ChecksumCRC32',
8789
digest: data => {
8890
const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
8991
return uint32ToBase64(new Crc32().update(input).digest() >>> 0); // >>> 0 coerce number to uint32
@@ -96,6 +98,7 @@ const algorithms = Object.freeze({
9698
createHash: () => new Crc32()
9799
},
98100
crc32c: {
101+
getObjectAttributeXMLTag: 'ChecksumCRC32C',
99102
digest: data => {
100103
const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
101104
return uint32ToBase64(new Crc32c().update(input).digest() >>> 0); // >>> 0 coerce number to uint32
@@ -105,6 +108,7 @@ const algorithms = Object.freeze({
105108
createHash: () => new Crc32c()
106109
},
107110
sha1: {
111+
getObjectAttributeXMLTag: 'ChecksumSHA1',
108112
digest: data => {
109113
const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
110114
return crypto.createHash('sha1').update(input).digest('base64');
@@ -114,6 +118,7 @@ const algorithms = Object.freeze({
114118
createHash: () => crypto.createHash('sha1')
115119
},
116120
sha256: {
121+
getObjectAttributeXMLTag: 'ChecksumSHA256',
117122
digest: data => {
118123
const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
119124
return crypto.createHash('sha256').update(input).digest('base64');

‎lib/api/apiUtils/object/objectAttributes.js‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
const { errorInstances } = require('arsenal');
22
const { getPartCountFromMd5 } = require('./partInfo');
3+
const { algorithms } = require('../integrity/validateChecksums');
34

45
/**
56
* Parse and validate attribute headers from a request.
@@ -81,6 +82,19 @@ function buildAttributesXml(objectMD, userMetadata, requestedAttrs, xml) {
8182
case 'ObjectSize':
8283
xml.push(`<ObjectSize>${objectMD['content-length']}</ObjectSize>`);
8384
break;
85+
case 'Checksum': {
86+
const { checksum } = objectMD;
87+
if (checksum) {
88+
const tag = algorithms[checksum.checksumAlgorithm].getObjectAttributeXMLTag;
89+
xml.push(
90+
'<Checksum>',
91+
`<${tag}>${checksum.checksumValue}</${tag}>`,
92+
`<ChecksumType>${checksum.checksumType}</ChecksumType>`,
93+
'</Checksum>',
94+
);
95+
}
96+
break;
97+
}
8498
case 'RestoreStatus':
8599
xml.push('<RestoreStatus>');
86100
xml.push(`<IsRestoreInProgress>${!!objectMD.restoreStatus?.inProgress}</IsRestoreInProgress>`);

‎lib/api/objectGetAttributes.js‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -150,18 +150,6 @@ async function objectGetAttributes(authInfo, request, log, callback) {
150150
true,
151151
);
152152

153-
if (requestedAttrs.has('Checksum')) {
154-
log.debug('Checksum attribute requested but not implemented', {
155-
method: OBJECT_GET_ATTRIBUTES,
156-
bucket: bucketName,
157-
key: objectKey,
158-
versionId,
159-
});
160-
const err = errors.NotImplemented.customizeDescription('Checksum attribute is not implemented');
161-
err.responseHeaders = responseHeaders;
162-
throw err;
163-
}
164-
165153
pushMetric(OBJECT_GET_ATTRIBUTES, log, {
166154
authInfo,
167155
bucket: bucketName,

‎tests/functional/aws-node-sdk/test/object/objectGetAttributes.js‎

Lines changed: 116 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ const {
1212
const { GetObjectAttributesExtendedCommand } = require('@scality/cloudserverclient');
1313
const withV4 = require('../support/withV4');
1414
const BucketUtility = require('../../lib/utility/bucket-util');
15+
const { algorithms } = require('../../../../../lib/api/apiUtils/integrity/validateChecksums');
1516

1617
const bucket = 'testbucket';
1718
const key = 'testobject';
@@ -30,7 +31,9 @@ describe('objectGetAttributes', () => {
3031

3132
beforeEach(async () => {
3233
await s3.send(new CreateBucketCommand({ Bucket: bucket }));
33-
await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: body }));
34+
await s3.send(new PutObjectCommand({
35+
Bucket: bucket, Key: key, Body: body, ChecksumAlgorithm: 'CRC64NVME',
36+
}));
3437
});
3538

3639
afterEach(async () => {
@@ -119,18 +122,28 @@ describe('objectGetAttributes', () => {
119122
assert.strictEqual(data.ETag, expectedMD5);
120123
});
121124

122-
it('should fail with NotImplemented when Checksum is requested', async () => {
123-
try {
124-
await s3.send(new GetObjectAttributesCommand({
125-
Bucket: bucket,
126-
Key: key,
127-
ObjectAttributes: ['Checksum'],
128-
}));
129-
assert.fail('Expected NotImplemented error');
130-
} catch (err) {
131-
assert.strictEqual(err.name, 'NotImplemented');
132-
assert.strictEqual(err.message, 'Checksum attribute is not implemented');
133-
}
125+
it('should return ChecksumCRC64NVME for object', async () => {
126+
const data = await s3.send(new GetObjectAttributesCommand({
127+
Bucket: bucket,
128+
Key: key,
129+
ObjectAttributes: ['Checksum'],
130+
}));
131+
132+
assert(data.Checksum, 'Checksum should be present');
133+
assert(data.Checksum.ChecksumCRC64NVME, 'ChecksumCRC64NVME should be present');
134+
assert.strictEqual(data.Checksum.ChecksumType, 'FULL_OBJECT');
135+
});
136+
137+
it('should not return Checksum when not requested', async () => {
138+
const data = await s3.send(new GetObjectAttributesCommand({
139+
Bucket: bucket,
140+
Key: key,
141+
ObjectAttributes: ['ETag', 'ObjectSize'],
142+
}));
143+
144+
assert(data.ETag, 'ETag should be present');
145+
assert(data.ObjectSize, 'ObjectSize should be present');
146+
assert.strictEqual(data.Checksum, undefined, 'Checksum should not be present');
134147
});
135148

136149
it("shouldn't return ObjectParts for non-MPU objects", async () => {
@@ -480,3 +493,93 @@ describe('objectGetAttributes with user metadata', () => {
480493
});
481494
});
482495
});
496+
497+
describe('objectGetAttributes with checksum', () => {
498+
withV4(sigCfg => {
499+
let bucketUtil;
500+
let s3;
501+
const checksumBucket = 'checksum-getattr-test';
502+
const checksumKey = 'checksum-test-object';
503+
const checksumBody = Buffer.from('checksum test body');
504+
505+
const expectedDigests = {};
506+
507+
before(async () => {
508+
bucketUtil = new BucketUtility('default', sigCfg);
509+
s3 = bucketUtil.s3;
510+
await s3.send(new CreateBucketCommand({ Bucket: checksumBucket }));
511+
512+
for (const [name, algo] of Object.entries(algorithms)) {
513+
expectedDigests[name] = await algo.digest(checksumBody);
514+
}
515+
});
516+
517+
after(async () => {
518+
await bucketUtil.empty(checksumBucket);
519+
await s3.send(new DeleteBucketCommand({ Bucket: checksumBucket }));
520+
});
521+
522+
Object.entries(algorithms).forEach(([name, { getObjectAttributeXMLTag }]) => {
523+
const sdkAlgorithm = name.toUpperCase();
524+
525+
it(`should return ${getObjectAttributeXMLTag} when object has ${name} checksum`, async () => {
526+
await s3.send(new PutObjectCommand({
527+
Bucket: checksumBucket,
528+
Key: checksumKey,
529+
Body: checksumBody,
530+
ChecksumAlgorithm: sdkAlgorithm,
531+
}));
532+
533+
const data = await s3.send(new GetObjectAttributesCommand({
534+
Bucket: checksumBucket,
535+
Key: checksumKey,
536+
ObjectAttributes: ['Checksum'],
537+
}));
538+
539+
assert(data.Checksum, 'Checksum should be present');
540+
assert.strictEqual(data.Checksum[getObjectAttributeXMLTag], expectedDigests[name]);
541+
assert.strictEqual(data.Checksum.ChecksumType, 'FULL_OBJECT');
542+
});
543+
544+
it(`should return ${getObjectAttributeXMLTag} along with other attributes`, async () => {
545+
await s3.send(new PutObjectCommand({
546+
Bucket: checksumBucket,
547+
Key: checksumKey,
548+
Body: checksumBody,
549+
ChecksumAlgorithm: sdkAlgorithm,
550+
}));
551+
552+
const data = await s3.send(new GetObjectAttributesCommand({
553+
Bucket: checksumBucket,
554+
Key: checksumKey,
555+
ObjectAttributes: ['ETag', 'Checksum', 'ObjectSize'],
556+
}));
557+
558+
assert(data.ETag, 'ETag should be present');
559+
assert(data.ObjectSize, 'ObjectSize should be present');
560+
assert(data.Checksum, 'Checksum should be present');
561+
assert.strictEqual(data.Checksum[getObjectAttributeXMLTag], expectedDigests[name]);
562+
assert.strictEqual(data.Checksum.ChecksumType, 'FULL_OBJECT');
563+
});
564+
});
565+
566+
it('should not return Checksum when not requested', async () => {
567+
await s3.send(new PutObjectCommand({
568+
Bucket: checksumBucket,
569+
Key: checksumKey,
570+
Body: checksumBody,
571+
ChecksumAlgorithm: 'CRC64NVME',
572+
}));
573+
574+
const data = await s3.send(new GetObjectAttributesCommand({
575+
Bucket: checksumBucket,
576+
Key: checksumKey,
577+
ObjectAttributes: ['ETag', 'ObjectSize'],
578+
}));
579+
580+
assert(data.ETag, 'ETag should be present');
581+
assert(data.ObjectSize, 'ObjectSize should be present');
582+
assert.strictEqual(data.Checksum, undefined, 'Checksum should not be present');
583+
});
584+
});
585+
});

‎tests/unit/api/apiUtils/object/objectAttributes.js‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ const {
33
parseAttributesHeaders,
44
buildAttributesXml
55
} = require('../../../../../lib/api/apiUtils/object/objectAttributes');
6+
const { algorithms } = require('../../../../../lib/api/apiUtils/integrity/validateChecksums');
67

78
const headerName = 'x-amz-object-attributes';
89
const allowedAttributes = new Set(['ETag', 'StorageClass', 'ObjectSize']);
@@ -156,6 +157,45 @@ describe('buildXmlAttributes', () => {
156157
});
157158
});
158159

160+
describe('with Checksum attribute', () => {
161+
const testData = Buffer.from('test data');
162+
const expectedDigests = {};
163+
164+
before(async () => {
165+
await Promise.all(Object.keys(algorithms).map(async name => {
166+
expectedDigests[name] = await algorithms[name].digest(testData);
167+
}));
168+
});
169+
170+
it('should not generate Checksum XML when checksum is absent', () => {
171+
const result = [];
172+
buildAttributesXml(objectMD, {}, ['Checksum'], result);
173+
174+
assert.strictEqual(result.length, 0);
175+
});
176+
177+
Object.entries(algorithms).forEach(([algo, { getObjectAttributeXMLTag }]) => {
178+
it(`should generate correct Checksum XML for ${algo}`, () => {
179+
const digest = expectedDigests[algo];
180+
const result = [];
181+
const md = {
182+
checksum: {
183+
checksumAlgorithm: algo,
184+
checksumValue: digest,
185+
checksumType: 'FULL_OBJECT',
186+
},
187+
};
188+
buildAttributesXml(md, {}, ['Checksum'], result);
189+
190+
assert.strictEqual(result.length, 4);
191+
assert.strictEqual(result[0], '<Checksum>');
192+
assert.strictEqual(result[1], `<${getObjectAttributeXMLTag}>${digest}</${getObjectAttributeXMLTag}>`);
193+
assert.strictEqual(result[2], '<ChecksumType>FULL_OBJECT</ChecksumType>');
194+
assert.strictEqual(result[3], '</Checksum>');
195+
});
196+
});
197+
});
198+
159199
describe('with object attributes and user metadata', () => {
160200
it('should build a comprehensive XML array with all supported features', () => {
161201
const result = [];

0 commit comments

Comments
 (0)