Skip to content

Commit ff3d080

Browse files
leif-scalityleif-scality
committed
CLDSRV-898: reject Checksum<X> field on a default MPU checksum
1 parent 1e724f1 commit ff3d080

3 files changed

Lines changed: 202 additions & 61 deletions

File tree

‎lib/api/completeMultipartUpload.js‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,18 @@ function validatePerPartChecksums(jsonList, storedParts, mpuSplitter, mpuChecksu
8383

8484
const presentTags = allChecksumXmlTags.filter(tag => part[tag]);
8585

86+
// AWS rejects any per-part Checksum<X> field on a default MPU
87+
// (one created without an explicit ChecksumAlgorithm) with
88+
// InvalidPart — even when the value is correct and even when the
89+
// field matches the implicit default algorithm.
90+
if (mpuChecksum.isDefault && presentTags.length > 0) {
91+
return errorInstances.InvalidPart.customizeDescription(
92+
'One or more of the specified parts could not be found. ' +
93+
'The part may not have been uploaded, or the specified ' +
94+
"entity tag may not match the part's entity tag.",
95+
);
96+
}
97+
8698
for (const tag of presentTags) {
8799
if (tag !== expectedTag) {
88100
const algoLabel = tag.replace(/^Checksum/, '').toLowerCase();

‎tests/functional/aws-node-sdk/test/object/completeMpuChecksum.js‎

Lines changed: 121 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -134,53 +134,126 @@ describe('CompleteMultipartUpload final-object checksum', () =>
134134
});
135135
});
136136

137-
it('should return CRC64NVME/FULL_OBJECT on CompleteMPU response when CreateMPU sent no checksum headers', async () => {
138-
const key = `complete-default-${Date.now()}`;
139-
140-
const create = await s3.send(
141-
new CreateMultipartUploadCommand({
142-
Bucket: bucket,
143-
Key: key,
144-
}),
145-
);
146-
147-
const uploadPart = await s3.send(
148-
new UploadPartCommand({
149-
Bucket: bucket,
150-
Key: key,
151-
UploadId: create.UploadId,
152-
PartNumber: 1,
153-
Body: partBody,
154-
}),
155-
);
156-
157-
const complete = await s3.send(
158-
new CompleteMultipartUploadCommand({
159-
Bucket: bucket,
160-
Key: key,
161-
UploadId: create.UploadId,
162-
MultipartUpload: {
163-
Parts: [{ PartNumber: 1, ETag: uploadPart.ETag }],
164-
},
165-
}),
166-
);
167-
168-
assert(
169-
complete.ChecksumCRC64NVME,
170-
`expected ChecksumCRC64NVME for default MPU, got: ${JSON.stringify(complete)}`,
171-
);
172-
assert.strictEqual(complete.ChecksumType, 'FULL_OBJECT');
173-
174-
// Default MPU is FULL_OBJECT — checksum is persisted, so
175-
// HeadObject must return the same value.
176-
const head = await s3.send(
177-
new HeadObjectCommand({
178-
Bucket: bucket,
179-
Key: key,
180-
ChecksumMode: 'ENABLED',
181-
}),
182-
);
183-
assert.strictEqual(head.ChecksumCRC64NVME, complete.ChecksumCRC64NVME);
184-
assert.strictEqual(head.ChecksumType, 'FULL_OBJECT');
137+
it(
138+
'should return CRC64NVME/FULL_OBJECT on CompleteMPU response ' + 'when CreateMPU sent no checksum headers',
139+
async () => {
140+
const key = `complete-default-${Date.now()}`;
141+
142+
const create = await s3.send(
143+
new CreateMultipartUploadCommand({
144+
Bucket: bucket,
145+
Key: key,
146+
}),
147+
);
148+
149+
const uploadPart = await s3.send(
150+
new UploadPartCommand({
151+
Bucket: bucket,
152+
Key: key,
153+
UploadId: create.UploadId,
154+
PartNumber: 1,
155+
Body: partBody,
156+
}),
157+
);
158+
159+
const complete = await s3.send(
160+
new CompleteMultipartUploadCommand({
161+
Bucket: bucket,
162+
Key: key,
163+
UploadId: create.UploadId,
164+
MultipartUpload: {
165+
Parts: [{ PartNumber: 1, ETag: uploadPart.ETag }],
166+
},
167+
}),
168+
);
169+
170+
assert(
171+
complete.ChecksumCRC64NVME,
172+
`expected ChecksumCRC64NVME for default MPU, got: ${JSON.stringify(complete)}`,
173+
);
174+
assert.strictEqual(complete.ChecksumType, 'FULL_OBJECT');
175+
176+
// Default MPU is FULL_OBJECT — checksum is persisted, so
177+
// HeadObject must return the same value.
178+
const head = await s3.send(
179+
new HeadObjectCommand({
180+
Bucket: bucket,
181+
Key: key,
182+
ChecksumMode: 'ENABLED',
183+
}),
184+
);
185+
assert.strictEqual(head.ChecksumCRC64NVME, complete.ChecksumCRC64NVME);
186+
assert.strictEqual(head.ChecksumType, 'FULL_OBJECT');
187+
},
188+
);
189+
190+
// AWS S3 rejects any per-part
191+
// Checksum<X> field on a default MPU (one created without an
192+
// explicit ChecksumAlgorithm) with InvalidPart — even when the
193+
// field matches the implicit CRC64NVME algorithm and value.
194+
describe('default MPU rejects per-part Checksum fields', () => {
195+
async function setupDefaultMpu() {
196+
const key = `complete-default-rejects-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
197+
const create = await s3.send(new CreateMultipartUploadCommand({ Bucket: bucket, Key: key }));
198+
const uploadPart = await s3.send(
199+
new UploadPartCommand({
200+
Bucket: bucket,
201+
Key: key,
202+
UploadId: create.UploadId,
203+
PartNumber: 1,
204+
Body: partBody,
205+
}),
206+
);
207+
return { key, uploadId: create.UploadId, eTag: uploadPart.ETag };
208+
}
209+
210+
async function assertInvalidPart(promise) {
211+
let caught;
212+
try {
213+
await promise;
214+
} catch (err) {
215+
caught = err;
216+
}
217+
assert(caught, 'expected CompleteMPU to reject');
218+
assert.strictEqual(
219+
caught.name,
220+
'InvalidPart',
221+
`expected InvalidPart, got ${caught.name}: ${caught.message}`,
222+
);
223+
}
224+
225+
it('should return InvalidPart when Part includes matching ChecksumCRC64NVME (correct value)', async () => {
226+
const { key, uploadId, eTag } = await setupDefaultMpu();
227+
const crc64 = await algorithms.crc64nvme.digest(partBody);
228+
await assertInvalidPart(
229+
s3.send(
230+
new CompleteMultipartUploadCommand({
231+
Bucket: bucket,
232+
Key: key,
233+
UploadId: uploadId,
234+
MultipartUpload: {
235+
Parts: [{ PartNumber: 1, ETag: eTag, ChecksumCRC64NVME: crc64 }],
236+
},
237+
}),
238+
),
239+
);
240+
});
241+
242+
it('should return InvalidPart when Part includes a non-matching algorithm field', async () => {
243+
const { key, uploadId, eTag } = await setupDefaultMpu();
244+
const crc32 = await algorithms.crc32.digest(partBody);
245+
await assertInvalidPart(
246+
s3.send(
247+
new CompleteMultipartUploadCommand({
248+
Bucket: bucket,
249+
Key: key,
250+
UploadId: uploadId,
251+
MultipartUpload: {
252+
Parts: [{ PartNumber: 1, ETag: eTag, ChecksumCRC32: crc32 }],
253+
},
254+
}),
255+
),
256+
);
257+
});
185258
});
186259
}));

‎tests/unit/api/completeMultipartUpload.js‎

Lines changed: 69 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -40,17 +40,19 @@ const SAMPLE_DIGESTS = {
4040
sha256: ['YWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWE=', 'YmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmI='],
4141
};
4242

43-
// Every AWS-valid (algorithm, type) combination, plus the implicit default.
43+
// Every AWS-valid (algorithm, type) combination for an explicit-algorithm MPU.
4444
// See validateChecksums.getChecksumDataFromMPUHeaders for the source of truth.
45+
// The implicit-default MPU (isDefault=true) is tested separately because AWS
46+
// rejects any per-part Checksum<X> field on a default MPU with InvalidPart,
47+
// regardless of value or algorithm.
4548
const MATRIX = [
46-
{ algorithm: 'crc32', type: 'COMPOSITE', isDefault: false },
47-
{ algorithm: 'crc32', type: 'FULL_OBJECT', isDefault: false },
48-
{ algorithm: 'crc32c', type: 'COMPOSITE', isDefault: false },
49-
{ algorithm: 'crc32c', type: 'FULL_OBJECT', isDefault: false },
50-
{ algorithm: 'crc64nvme', type: 'FULL_OBJECT', isDefault: false },
51-
{ algorithm: 'crc64nvme', type: 'FULL_OBJECT', isDefault: true },
52-
{ algorithm: 'sha1', type: 'COMPOSITE', isDefault: false },
53-
{ algorithm: 'sha256', type: 'COMPOSITE', isDefault: false },
49+
{ algorithm: 'crc32', type: 'COMPOSITE' },
50+
{ algorithm: 'crc32', type: 'FULL_OBJECT' },
51+
{ algorithm: 'crc32c', type: 'COMPOSITE' },
52+
{ algorithm: 'crc32c', type: 'FULL_OBJECT' },
53+
{ algorithm: 'crc64nvme', type: 'FULL_OBJECT' },
54+
{ algorithm: 'sha1', type: 'COMPOSITE' },
55+
{ algorithm: 'sha256', type: 'COMPOSITE' },
5456
];
5557

5658
function makeStoredPart(partNumber, checksum) {
@@ -88,11 +90,11 @@ function pickWrongAlgo(algo) {
8890

8991
describe('validatePerPartChecksums', () => {
9092
describe('AWS combination matrix', () => {
91-
MATRIX.forEach(({ algorithm, type, isDefault }) => {
92-
const label = `${algorithm}/${type}${isDefault ? ' (default)' : ''}`;
93+
MATRIX.forEach(({ algorithm, type }) => {
94+
const label = `${algorithm}/${type}`;
9395
const tag = TAG_BY_ALGO[algorithm];
9496
const [d1, d2] = SAMPLE_DIGESTS[algorithm];
95-
const mpuChecksum = { algorithm, type, isDefault };
97+
const mpuChecksum = { algorithm, type, isDefault: false };
9698

9799
const stored = [makeStoredPart(1, { algorithm, value: d1 }), makeStoredPart(2, { algorithm, value: d2 })];
98100

@@ -143,7 +145,7 @@ describe('validatePerPartChecksums', () => {
143145
);
144146
});
145147

146-
const requiresPerPart = type === 'COMPOSITE' && !isDefault;
148+
const requiresPerPart = type === 'COMPOSITE';
147149
const missingLabel = requiresPerPart
148150
? 'should return InvalidRequest when a part is missing its checksum'
149151
: 'should accept a parts list missing per-part checksums';
@@ -165,6 +167,60 @@ describe('validatePerPartChecksums', () => {
165167
});
166168
});
167169

170+
describe('default MPU (isDefault=true)', () => {
171+
// AWS S3 rejects any per-part
172+
// Checksum<X> field on a default MPU with InvalidPart — even when
173+
// the field matches the implicit CRC64NVME algorithm and the value
174+
// is the same one the part was stored with.
175+
const mpuChecksum = { algorithm: 'crc64nvme', type: 'FULL_OBJECT', isDefault: true };
176+
const [d1, d2] = SAMPLE_DIGESTS.crc64nvme;
177+
const stored = [
178+
makeStoredPart(1, { algorithm: 'crc64nvme', value: d1 }),
179+
makeStoredPart(2, { algorithm: 'crc64nvme', value: d2 }),
180+
];
181+
const invalidPartMessage =
182+
'One or more of the specified parts could not be ' +
183+
'found. The part may not have been uploaded, or ' +
184+
'the specified entity tag may not match the ' +
185+
"part's entity tag.";
186+
187+
it('should accept when no parts include a checksum field', () => {
188+
const jsonList = { Part: [makeJsonPart(1, 'etag1'), makeJsonPart(2, 'etag2')] };
189+
const err = validatePerPartChecksums(jsonList, stored, SPLITTER, mpuChecksum);
190+
assert.strictEqual(err, null);
191+
});
192+
193+
it('should return InvalidPart when a part includes the matching field (correct value)', () => {
194+
const jsonList = {
195+
Part: [makeJsonPart(1, 'etag1', { ChecksumCRC64NVME: d1 }), makeJsonPart(2, 'etag2')],
196+
};
197+
const err = validatePerPartChecksums(jsonList, stored, SPLITTER, mpuChecksum);
198+
assert(err);
199+
assert.strictEqual(err.is.InvalidPart, true);
200+
assert.strictEqual(err.description, invalidPartMessage);
201+
});
202+
203+
it('should return InvalidPart when a part includes the matching field (wrong value)', () => {
204+
const jsonList = {
205+
Part: [makeJsonPart(1, 'etag1', { ChecksumCRC64NVME: d2 }), makeJsonPart(2, 'etag2')],
206+
};
207+
const err = validatePerPartChecksums(jsonList, stored, SPLITTER, mpuChecksum);
208+
assert(err);
209+
assert.strictEqual(err.is.InvalidPart, true);
210+
assert.strictEqual(err.description, invalidPartMessage);
211+
});
212+
213+
it('should return InvalidPart when a part includes a non-matching algorithm field', () => {
214+
const jsonList = {
215+
Part: [makeJsonPart(1, 'etag1', { ChecksumCRC32: SAMPLE_DIGESTS.crc32[0] }), makeJsonPart(2, 'etag2')],
216+
};
217+
const err = validatePerPartChecksums(jsonList, stored, SPLITTER, mpuChecksum);
218+
assert(err);
219+
assert.strictEqual(err.is.InvalidPart, true);
220+
assert.strictEqual(err.description, invalidPartMessage);
221+
});
222+
});
223+
168224
describe('legacy MPU (no algorithm configured)', () => {
169225
// Pre-feature MPUs have storedMetadata.checksumAlgorithm === undefined.
170226
// Pre-PR CompleteMPU silently ignored any per-part Checksum<X> body

0 commit comments

Comments
 (0)