Skip to content

Commit e519604

Browse files
leif-scalityleif-scality
committed
CLDSRV-848: check x-amz-checksum-[crc64nvme, crc32, crc32C, sha1, sha256] headers
1 parent 5889490 commit e519604

File tree

3 files changed

+144
-19
lines changed

3 files changed

+144
-19
lines changed

‎lib/Config.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -556,6 +556,7 @@ function parseIntegrityChecks(config) {
556556
'bucketPutReplication': true,
557557
'bucketPutVersioning': true,
558558
'bucketPutWebsite': true,
559+
'bucketPutLogging': true,
559560
'multiObjectDelete': true,
560561
'objectPutACL': true,
561562
'objectPutLegalHold': true,

‎lib/api/api.js‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -300,15 +300,19 @@ const api = {
300300
}
301301

302302
const buff = Buffer.concat(post, bodyLength);
303+
return validateMethodChecksumNoChunking(request, buff, log)
304+
.then(error => {
305+
if (error) {
306+
return next(error);
307+
}
303308

304-
const err = validateMethodChecksumNoChunking(request, buff, log);
305-
if (err) {
306-
return next(err);
307-
}
308-
309-
// Convert array of post buffers into one string
310-
request.post = buff.toString();
311-
return next(null, userInfo, authorizationResults, streamingV4Params, infos);
309+
// Convert array of post buffers into one string
310+
request.post = buff.toString();
311+
return next(null, userInfo, authorizationResults, streamingV4Params, infos);
312+
})
313+
.catch(error => {
314+
next(error);
315+
});
312316
});
313317
return undefined;
314318
},

‎lib/api/apiUtils/integrity/validateChecksums.js‎

Lines changed: 131 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,157 @@
11
const crypto = require('crypto');
2+
const { Crc32 } = require('@aws-crypto/crc32');
3+
const { Crc32c } = require('@aws-crypto/crc32c');
4+
const { CrtCrc64Nvme } = require('@aws-sdk/crc64-nvme-crt');
25
const { errors: ArsenalErrors } = require('arsenal');
36
const { config } = require('../../../Config');
47

58
const ChecksumError = Object.freeze({
69
MD5Mismatch: 'MD5Mismatch',
10+
XAmzMismatch: 'XAmzMismatch',
711
MissingChecksum: 'MissingChecksum',
12+
AlgoNotSupported: 'AlgoNotSupported',
13+
MultipleChecksumTypes: 'MultipleChecksumTypes',
14+
MissingCorresponding: 'MissingCorresponding'
815
});
916

17+
// TEMP
18+
// https://github.com/aws/aws-sdk-js-v3/issues/6744
19+
// https://stackoverflow.com/questions/77663519/
20+
// does-aws-s3-allow-specifying-multiple-checksum-values-crc32-crc32c-sha1-and-s
21+
// Expecting a single x-amz-checksum- header. Multiple checksum Types are not allowed.
22+
// XAmzContentChecksumMismatch: The provided 'x-amz-checksum' header does not match what was computed.
23+
24+
// TODO:
25+
// x-amz-checksum-algorithm x2 different
26+
// x-amz-checksum-algorithm x2 equal
27+
// x-amz-checksum-algorithm x2 + x-amz-checksum- valid x2
28+
// x-amz-checksum-algorithm x2 + x-amz-checksum- invalid x2
29+
// x-amz-checksum-algorithm x2 + x-amz-checksum- mismatch x2
30+
31+
// What to do about the security vuln package?
32+
// Should we push only to 9.3???
33+
34+
const algorithms = {
35+
'crc64nvme': async data => {
36+
const crc = new CrtCrc64Nvme();
37+
crc.update(data);
38+
const result = await crc.digest();
39+
return Buffer.from(result).toString('base64');
40+
},
41+
'crc32': data => uint32ToBase64(new Crc32().update(data).digest() >>> 0),
42+
'crc32c': data => uint32ToBase64(new Crc32c().update(data).digest() >>> 0),
43+
'sha1': data => crypto.createHash('sha1').update(data).digest('base64'),
44+
'sha256': data => crypto.createHash('sha256').update(data).digest('base64'),
45+
};
46+
47+
function uint32ToBase64(num) {
48+
const buf = Buffer.alloc(4);
49+
buf.writeUInt32BE(num, 0);
50+
return buf.toString('base64');
51+
}
52+
53+
async function validateXAmzChecksums(headers, body) {
54+
const checksumHeaders = Object.keys(headers).filter(header => header.startsWith('x-amz-checksum-'));
55+
const xAmzCheckumCnt = checksumHeaders.length;
56+
// console.log(headers, body);
57+
if (xAmzCheckumCnt > 1) {
58+
return { error: ChecksumError.MultipleChecksumTypes, details: null };
59+
}
60+
61+
if ('x-amz-sdk-checksum-algorithm' in headers) {
62+
let algo = headers['x-amz-sdk-checksum-algorithm'];
63+
if (typeof algo !== 'string') {
64+
return { error: ChecksumError.AlgoNotSupported, details: null }; // What if invalid algo like a number?
65+
}
66+
67+
algo = algo.toLowerCase();
68+
if (algo in algorithms === false) {
69+
return { error: ChecksumError.AlgoNotSupported, details: null };
70+
}
71+
72+
if (`x-amz-checksum-${algo}` in headers === false) {
73+
return { error: ChecksumError.MissingCorresponding, details: null };
74+
}
75+
76+
const expected = headers[`x-amz-checksum-${algo}`];
77+
const calculated = await algorithms[algo](body);
78+
// console.log('EXPECTED:', expected, calculated);
79+
if (expected !== calculated) {
80+
return { error: ChecksumError.XAmzMismatch, details: null };
81+
}
82+
83+
return null;
84+
}
85+
86+
if (xAmzCheckumCnt === 0) {
87+
return { error: ChecksumError.MissingChecksum, details: null };
88+
}
89+
90+
// No x-amz-sdk-checksum-algorithm we expect one x-amz-checksum-[crc64nvme, crc32, crc32C, sha1, sha256].
91+
let algo = checksumHeaders[0].split('-')[3];
92+
if (typeof algo !== 'string') {
93+
return { error: ChecksumError.AlgoNotSupported, details: null }; // What if invalid algo like a number?
94+
}
95+
96+
algo = algo.toLowerCase();
97+
if (algo in algorithms === false) {
98+
return { error: ChecksumError.AlgoNotSupported, details: null };;
99+
}
100+
101+
const expected = headers[`x-amz-checksum-${algo}`];
102+
const calculated = await algorithms[algo](body);
103+
if (expected !== calculated) {
104+
return { error: ChecksumError.XAmzMismatch, details: null };
105+
}
106+
107+
return null;
108+
}
109+
10110
/**
11111
* validateChecksumsNoChunking - Validate the checksums of a request.
12112
* @param {object} headers - http headers
13113
* @param {Buffer} body - http request body
14114
* @return {object} - error
15115
*/
16-
function validateChecksumsNoChunking(headers, body) {
17-
if (headers && 'content-md5' in headers) {
116+
async function validateChecksumsNoChunking(headers, body) {
117+
if (!headers) {
118+
return { error: ChecksumError.MissingChecksum, details: null };
119+
}
120+
121+
let md5Present = false;
122+
if ('content-md5' in headers) {
18123
const md5 = crypto.createHash('md5').update(body).digest('base64');
19124
if (md5 !== headers['content-md5']) {
20125
return { error: ChecksumError.MD5Mismatch, details: { calculated: md5, expected: headers['content-md5'] } };
21126
}
22127

23-
return null;
128+
md5Present = true;
129+
}
130+
131+
const err = await validateXAmzChecksums(headers, body);
132+
if (err && err.error === ChecksumError.MissingChecksum && !md5Present) {
133+
// Return MissingChecksum only if no MD5 and no x-amz-checksum-.
134+
return { error: ChecksumError.MissingChecksum, details: null };
135+
}
136+
137+
return err;
138+
}
139+
140+
async function defaultValidationFunc2(request, body, log) { // Rename
141+
const err = await validateChecksumsNoChunking(request.headers, body);
142+
if (err) {
143+
log.debug('failed checksum validation', { method: request.apiMethod, error: err });
144+
return ArsenalErrors.BadDigest; // FIXME: InvalidDigest vs BadDigest
24145
}
25146

26-
return { error: ChecksumError.MissingChecksum, details: null };
147+
return null;
27148
}
28149

29-
function defaultValidationFunc(request, body, log) {
30-
const err = validateChecksumsNoChunking(request.headers, body);
150+
async function defaultValidationFunc(request, body, log) { // Rename
151+
const err = await validateChecksumsNoChunking(request.headers, body);
31152
if (err && err.error !== ChecksumError.MissingChecksum) {
32153
log.debug('failed checksum validation', { method: request.apiMethod }, err);
33-
return ArsenalErrors.BadDigest;
154+
return ArsenalErrors.BadDigest; // FIXME: InvalidDigest vs BadDigest
34155
}
35156

36157
return null;
@@ -62,14 +183,13 @@ const methodValidationFunc = Object.freeze({
62183
* @param {object} log - logger
63184
* @return {object} - error
64185
*/
65-
function validateMethodChecksumNoChunking(request, body, log) {
186+
async function validateMethodChecksumNoChunking(request, body, log) {
66187
if (config.integrityChecks[request.apiMethod]) {
67188
const validationFunc = methodValidationFunc[request.apiMethod];
68189
if (!validationFunc) {
69-
return null;
190+
return await defaultValidationFunc2(request, body, log);
70191
}
71-
72-
return validationFunc(request, body, log);
192+
return await validationFunc(request, body, log);
73193
}
74194

75195
return null;

0 commit comments

Comments
 (0)