Skip to content

Commit 2ca1c1d

Browse files
leif-scalityclaude
leif-scality
and
claude
committed
fixup! CLDSRV-892: UploadPart store part checksum
fixup! CLDSRV-892: UploadPart store part checksum Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 7956a1f commit 2ca1c1d

2 files changed

Lines changed: 215 additions & 0 deletions

File tree

‎tests/functional/aws-node-sdk/test/object/mpuUploadPartChecksum.js‎

Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
const assert = require('assert');
2+
const {
3+
CreateBucketCommand,
4+
CreateMultipartUploadCommand,
5+
AbortMultipartUploadCommand,
6+
UploadPartCommand,
7+
DeleteBucketCommand,
8+
} = require('@aws-sdk/client-s3');
9+
10+
const withV4 = require('../support/withV4');
11+
const BucketUtility = require('../../lib/utility/bucket-util');
12+
const { algorithms } = require('../../../../../lib/api/apiUtils/integrity/validateChecksums');
13+
14+
const bucket = `mpu-part-checksum-test-${Date.now()}`;
15+
const key = 'test-part-checksum-key';
16+
const partBody = Buffer.from('I am a part body for checksum testing', 'utf8');
17+
18+
const allAlgos = ['CRC32', 'CRC32C', 'SHA1', 'SHA256'];
19+
20+
// Maps algo name to the UploadPartCommand checksum field name
21+
const checksumField = {
22+
CRC32: 'ChecksumCRC32',
23+
CRC32C: 'ChecksumCRC32C',
24+
SHA1: 'ChecksumSHA1',
25+
SHA256: 'ChecksumSHA256',
26+
};
27+
28+
// Pre-compute correct digests for partBody
29+
const correctDigest = {};
30+
// A valid-length but incorrect digest for each algo
31+
const wrongDigest = {};
32+
33+
before(async () => {
34+
for (const algo of allAlgos) {
35+
36+
correctDigest[algo] = await algorithms[algo.toLowerCase()].digest(partBody);
37+
}
38+
// Generate wrong digests: flip the first character
39+
for (const algo of allAlgos) {
40+
const correct = correctDigest[algo];
41+
const flipped = correct[0] === 'A' ? `B${correct.slice(1)}` : `A${correct.slice(1)}`;
42+
wrongDigest[algo] = flipped;
43+
}
44+
});
45+
46+
describe('UploadPart checksum validation', () =>
47+
withV4(sigCfg => {
48+
let bucketUtil;
49+
let s3;
50+
51+
before(async () => {
52+
bucketUtil = new BucketUtility('default', sigCfg);
53+
s3 = bucketUtil.s3;
54+
await s3.send(new CreateBucketCommand({ Bucket: bucket }));
55+
});
56+
57+
after(async () => {
58+
await bucketUtil.empty(bucket);
59+
await s3.send(new DeleteBucketCommand({ Bucket: bucket }));
60+
});
61+
62+
// For each non-default MPU algo, test that:
63+
// - matching algo with correct digest succeeds
64+
// - matching algo with wrong digest fails with BadDigest
65+
// - every other algo is rejected with InvalidRequest
66+
// - no checksum header is accepted
67+
allAlgos.forEach(mpuAlgo => {
68+
describe(`MPU created with ${mpuAlgo}`, () => {
69+
let uploadId;
70+
let partNum = 0;
71+
72+
before(async () => {
73+
const res = await s3.send(new CreateMultipartUploadCommand({
74+
Bucket: bucket, Key: key,
75+
ChecksumAlgorithm: mpuAlgo,
76+
}));
77+
uploadId = res.UploadId;
78+
});
79+
80+
after(async () => {
81+
await s3.send(new AbortMultipartUploadCommand({
82+
Bucket: bucket, Key: key, UploadId: uploadId,
83+
}));
84+
});
85+
86+
it(`should accept ${mpuAlgo} with correct digest`, async () => {
87+
partNum++;
88+
const res = await s3.send(new UploadPartCommand({
89+
Bucket: bucket, Key: key, UploadId: uploadId,
90+
PartNumber: partNum, Body: partBody,
91+
[checksumField[mpuAlgo]]: correctDigest[mpuAlgo],
92+
}));
93+
assert.strictEqual(res[checksumField[mpuAlgo]], correctDigest[mpuAlgo]);
94+
});
95+
96+
it(`should reject ${mpuAlgo} with wrong digest (BadDigest)`, async () => {
97+
partNum++;
98+
try {
99+
await s3.send(new UploadPartCommand({
100+
Bucket: bucket, Key: key, UploadId: uploadId,
101+
PartNumber: partNum, Body: partBody,
102+
[checksumField[mpuAlgo]]: wrongDigest[mpuAlgo],
103+
}));
104+
assert.fail('Expected BadDigest error');
105+
} catch (err) {
106+
assert.strictEqual(err.name, 'BadDigest');
107+
}
108+
});
109+
110+
// Note: AWS SDK v3 always sends a default crc32 checksum,
111+
// so "no checksum header" cannot be tested via the SDK for
112+
// non-default MPUs (it would be rejected as a mismatch).
113+
114+
allAlgos.filter(a => a !== mpuAlgo).forEach(otherAlgo => {
115+
it(`should reject ${otherAlgo} when MPU is ${mpuAlgo} (InvalidRequest)`, async () => {
116+
partNum++;
117+
try {
118+
await s3.send(new UploadPartCommand({
119+
Bucket: bucket, Key: key, UploadId: uploadId,
120+
PartNumber: partNum, Body: partBody,
121+
[checksumField[otherAlgo]]: correctDigest[otherAlgo],
122+
}));
123+
assert.fail('Expected InvalidRequest error');
124+
} catch (err) {
125+
assert.strictEqual(err.name, 'InvalidRequest');
126+
}
127+
});
128+
});
129+
});
130+
});
131+
132+
// Default MPU (no ChecksumAlgorithm) should accept any algo
133+
describe('MPU created with no checksum (default)', () => {
134+
let uploadId;
135+
let partNum = 0;
136+
137+
before(async () => {
138+
const res = await s3.send(new CreateMultipartUploadCommand({
139+
Bucket: bucket, Key: key,
140+
}));
141+
uploadId = res.UploadId;
142+
});
143+
144+
after(async () => {
145+
await s3.send(new AbortMultipartUploadCommand({
146+
Bucket: bucket, Key: key, UploadId: uploadId,
147+
}));
148+
});
149+
150+
allAlgos.forEach(algo => {
151+
it(`should accept ${algo} with correct digest`, async () => {
152+
partNum++;
153+
const res = await s3.send(new UploadPartCommand({
154+
Bucket: bucket, Key: key, UploadId: uploadId,
155+
PartNumber: partNum, Body: partBody,
156+
[checksumField[algo]]: correctDigest[algo],
157+
}));
158+
assert.strictEqual(res[checksumField[algo]], correctDigest[algo]);
159+
});
160+
161+
it(`should reject ${algo} with wrong digest (BadDigest)`, async () => {
162+
partNum++;
163+
try {
164+
await s3.send(new UploadPartCommand({
165+
Bucket: bucket, Key: key, UploadId: uploadId,
166+
PartNumber: partNum, Body: partBody,
167+
[checksumField[algo]]: wrongDigest[algo],
168+
}));
169+
assert.fail('Expected BadDigest error');
170+
} catch (err) {
171+
assert.strictEqual(err.name, 'BadDigest');
172+
}
173+
});
174+
});
175+
176+
it('should accept part with no checksum header', async () => {
177+
partNum++;
178+
const res = await s3.send(new UploadPartCommand({
179+
Bucket: bucket, Key: key, UploadId: uploadId,
180+
PartNumber: partNum, Body: partBody,
181+
}));
182+
assert(res.ETag);
183+
});
184+
});
185+
})
186+
);

‎tests/unit/api/objectPutPartChecksum.js‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,35 @@ describe('objectPutPart checksum validation', () => {
135135
});
136136
});
137137

138+
it('should return BadDigest when matching algo but wrong digest', done => {
139+
initiateMPU({ 'x-amz-checksum-algorithm': 'sha256' }, (err, uploadId) => {
140+
assert.ifError(err);
141+
// Algo matches MPU (sha256) but digest is wrong
142+
const request = makePutPartRequest(uploadId, 1, partBody, {
143+
'x-amz-checksum-sha256': 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=',
144+
});
145+
objectPutPart(authInfo, request, undefined, log, err => {
146+
assert(err, 'Expected an error');
147+
assert.strictEqual(err.message, 'BadDigest');
148+
done();
149+
});
150+
});
151+
});
152+
153+
it('should return InvalidRequest when MPU algo is sha256 and part sends crc32', done => {
154+
initiateMPU({ 'x-amz-checksum-algorithm': 'sha256' }, (err, uploadId) => {
155+
assert.ifError(err);
156+
const request = makePutPartRequest(uploadId, 1, partBody, {
157+
'x-amz-checksum-crc32': 'DUoRhQ==',
158+
});
159+
objectPutPart(authInfo, request, undefined, log, err => {
160+
assert(err, 'Expected an error');
161+
assert.strictEqual(err.message, 'InvalidRequest');
162+
done();
163+
});
164+
});
165+
});
166+
138167
it('should accept any checksum algo on default (no algo specified) MPU', done => {
139168
initiateMPU({}, (err, uploadId) => {
140169
assert.ifError(err);

0 commit comments

Comments
 (0)