Skip to content

Commit 46ccab1

Browse files
leif-scalityleif-scality
committed
CLDSRV-780: update ServerAccessLogger to follow json schema
1 parent 39709ab commit 46ccab1

4 files changed

Lines changed: 91 additions & 82 deletions

File tree

‎lib/api/bucketGet.js‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -292,10 +292,18 @@ function bucketGet(authInfo, request, log, callback) {
292292
log.addDefaultFields({
293293
action: 'ListObjectsV2',
294294
});
295+
if (request.serverAccessLog) {
296+
// eslint-disable-next-line no-param-reassign
297+
request.serverAccessLog.analyticsAction = 'ListObjectsV2';
298+
}
295299
} else if (params.versions !== undefined) {
296300
log.addDefaultFields({
297301
action: 'ListObjectVersions',
298302
});
303+
if (request.serverAccessLog) {
304+
// eslint-disable-next-line no-param-reassign
305+
request.serverAccessLog.analyticsAction = 'ListObjectVersions';
306+
}
299307
}
300308
log.debug('processing request', { method: 'bucketGet' });
301309
const encoding = params['encoding-type'];

‎lib/utilities/serverAccessLogger.js‎

Lines changed: 72 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -182,74 +182,75 @@ function getRemoteIPFromRequest(request) {
182182
}
183183

184184
function getOperation(req) {
185+
// https://stackoverflow.com/questions/42707878/amazon-s3-logs-operation-definition
185186
const methodToResType = Object.freeze({
186187
'bucketDelete': 'BUCKET',
187-
'bucketDeleteCors': 'BUCKET',
188-
'bucketDeleteEncryption': 'BUCKET',
189-
'bucketDeleteWebsite': 'BUCKET',
188+
'bucketDeleteCors': 'CORS',
189+
'bucketDeleteEncryption': 'ENCRYPTION',
190+
'bucketDeleteWebsite': 'WEBSITE',
190191
'bucketGet': 'BUCKET',
191-
'bucketGetACL': 'BUCKET',
192-
'bucketGetCors': 'BUCKET',
193-
'bucketGetObjectLock': 'BUCKET',
192+
'bucketGetACL': 'ACL',
193+
'bucketGetCors': 'CORS',
194+
'bucketGetObjectLock': 'OBJECT',
194195
'bucketGetVersioning': 'VERSIONING',
195-
'bucketGetWebsite': 'BUCKET',
196-
'bucketGetLocation': 'BUCKET',
197-
'bucketGetEncryption': 'BUCKET',
196+
'bucketGetWebsite': 'WEBSITE',
197+
'bucketGetLocation': 'LOCATION',
198+
'bucketGetEncryption': 'ENCRYPTION',
198199
'bucketHead': 'BUCKET',
199200
'bucketPut': 'BUCKET',
200-
'bucketPutACL': 'BUCKET',
201-
'bucketPutCors': 'BUCKET',
201+
'bucketPutACL': 'ACL',
202+
'bucketPutCors': 'CORS',
202203
'bucketPutVersioning': 'VERSIONING',
203-
'bucketPutTagging': 'BUCKET',
204-
'bucketDeleteTagging': 'BUCKET',
205-
'bucketGetTagging': 'BUCKET',
206-
'bucketPutWebsite': 'BUCKET',
207-
'bucketPutReplication': 'BUCKET',
208-
'bucketGetReplication': 'BUCKET',
209-
'bucketDeleteReplication': 'BUCKET',
210-
'bucketDeleteQuota': 'BUCKET',
211-
'bucketPutLifecycle': 'BUCKET',
212-
'bucketUpdateQuota': 'BUCKET',
213-
'bucketGetLifecycle': 'BUCKET',
214-
'bucketDeleteLifecycle': 'BUCKET',
204+
'bucketPutTagging': 'TAGGING',
205+
'bucketDeleteTagging': 'TAGGING',
206+
'bucketGetTagging': 'TAGGING',
207+
'bucketPutWebsite': 'WEBSITE',
208+
'bucketPutReplication': 'REPLICATION',
209+
'bucketGetReplication': 'REPLICATION',
210+
'bucketDeleteReplication': 'REPLICATION',
211+
'bucketDeleteQuota': 'QUOTA',
212+
'bucketPutLifecycle': 'LIFECYCLE',
213+
'bucketUpdateQuota': 'QUOTA',
214+
'bucketGetLifecycle': 'LIFECYCLE',
215+
'bucketDeleteLifecycle': 'LIFECYCLE',
215216
'bucketPutPolicy': 'BUCKETPOLICY',
216217
'bucketGetPolicy': 'BUCKETPOLICY',
217-
'bucketGetQuota': 'BUCKET',
218+
'bucketGetQuota': 'QUOTA',
218219
'bucketDeletePolicy': 'BUCKETPOLICY',
219-
'bucketPutObjectLock': 'BUCKET',
220-
'bucketPutNotification': 'BUCKET',
221-
'bucketGetNotification': 'BUCKET',
222-
'bucketPutEncryption': 'BUCKET',
220+
'bucketPutObjectLock': 'OBJECTLOCK',
221+
'bucketPutNotification': 'NOTIFICATION',
222+
'bucketGetNotification': 'NOTIFICATION',
223+
'bucketPutEncryption': 'ENCRYPTION',
223224
'bucketPutLogging': 'LOGGING_STATUS',
224225
'bucketGetLogging': 'LOGGING_STATUS',
225226
// 'corsPreflight': '',
226-
'completeMultipartUpload': 'OBJECT',
227-
'initiateMultipartUpload': 'OBJECT',
228-
'listMultipartUploads': 'OBJECT',
229-
'listParts': 'OBJECT',
227+
'completeMultipartUpload': 'UPLOAD',
228+
'initiateMultipartUpload': 'UPLOAD',
229+
'listMultipartUploads': 'UPLOADS',
230+
'listParts': 'UPLOAD',
230231
'metadataSearch': 'OBJECT',
231232
'multiObjectDelete': 'OBJECT',
232-
'multipartDelete': 'OBJECT',
233+
'multipartDelete': 'UPLOAD',
233234
'objectDelete': 'OBJECT',
234-
'objectDeleteTagging': 'OBJECT',
235+
'objectDeleteTagging': 'TAGGING',
235236
'objectGet': 'OBJECT',
236-
'objectGetACL': 'OBJECT',
237-
'objectGetLegalHold': 'OBJECT',
238-
'objectGetRetention': 'OBJECT',
239-
'objectGetTagging': 'OBJECT',
240-
'objectCopy': 'OBJECT',
237+
'objectGetACL': 'ACL',
238+
'objectGetLegalHold': 'LEGAL_HOLD',
239+
'objectGetRetention': 'RETENTION',
240+
'objectGetTagging': 'TAGGING',
241+
'objectCopy': 'COPY',
241242
'objectHead': 'OBJECT',
242243
'objectPut': 'OBJECT',
243-
'objectPutACL': 'OBJECT',
244-
'objectPutLegalHold': 'OBJECT',
245-
'objectPutTagging': 'OBJECT',
246-
'objectPutPart': 'OBJECT',
247-
'objectPutCopyPart': 'OBJECT',
248-
'objectPutRetention': 'OBJECT',
244+
'objectPutACL': 'ACL',
245+
'objectPutLegalHold': 'LEGAL_HOLD',
246+
'objectPutTagging': 'TAGGING',
247+
'objectPutPart': 'PART',
248+
'objectPutCopyPart': 'COPY',
249+
'objectPutRetention': 'RETENTION',
249250
'objectRestore': 'OBJECT',
250-
// 'serviceGet': '',
251-
// 'websiteGet': '',
252-
// 'websiteHead': '',
251+
'serviceGet': 'SERVICE', // ListBuckets
252+
'websiteGet': 'WEBSITE',
253+
'websiteHead': 'WEBSITE',
253254
});
254255

255256
return `REST.${req.method}.${methodToResType[req.apiMethod] ? methodToResType[req.apiMethod] : 'UNKNOWN'}`;
@@ -301,7 +302,7 @@ function getObjectSize(request, response) {
301302
return null;
302303
}
303304

304-
return len;
305+
return Number(len);
305306
}
306307

307308
if (request && objectSizePutMethods[request.apiMethod]) {
@@ -310,7 +311,7 @@ function getObjectSize(request, response) {
310311
return null;
311312
}
312313

313-
return len;
314+
return Number(len);
314315
}
315316

316317
return null;
@@ -387,58 +388,58 @@ function logServerAccess(req, res) {
387388
pid: process.pid,
388389
389390
// Analytics
390-
action: params.analyticsAction || null,
391-
accountName: params.analyticsAccountName || null,
391+
action: params.analyticsAction === undefined ? null : params.analyticsAction,
392+
accountName: params.analyticsAccountName === undefined ? null : params.analyticsAccountName,
392393
accountDisplayName: authInfo ? authInfo.getAccountDisplayName() : null,
393-
userName: params.analyticsUserName || null,
394-
clientPort: req.socket.remotePort || null,
395-
httpMethod: req.method || null,
396-
bytesDeleted: params.analyticsBytesDeleted || params.analyticsBytesDeleted === 0 ? 0 : null,
397-
bytesReceived: req.parsedContentLength || 0,
398-
bodyLength: parseInt(req.headers['content-length'], 10) || 0,
399-
contentLength: req.parsedContentLength || 0,
394+
userName: params.analyticsUserName === undefined ? null : params.analyticsUserName,
395+
clientPort: req.socket.remotePort === undefined ? null : req.socket.remotePort,
396+
httpMethod: req.method === undefined ? null : req.method,
397+
bytesDeleted: params.analyticsBytesDeleted === undefined ? null : params.analyticsBytesDeleted,
398+
bytesReceived: req.parsedContentLength === undefined ? null : req.parsedContentLength,
399+
bodyLength: req.headers['content-length'] === undefined ? null : parseInt(req.headers['content-length'], 10),
400+
contentLength: req.parsedContentLength === undefined ? null : req.parsedContentLength,
400401
// eslint-disable-next-line camelcase
401402
elapsed_ms: calculateElapsedMS(params.startTime, params.onCloseEndTime),
402-
httpURL: req.url || null,
403+
httpURL: req.url === undefined ? null : req.url,
403404
404405
// AWS access server logs fields https://docs.aws.amazon.com/AmazonS3/latest/userguide/LogFormat.html
405406
startTime: timestampToDateTime643(params.startTimeUnixMS), // AWS "Time" field
406407
requester: getRequester(authInfo),
407408
operation: getOperation(req),
408409
requestURI: getURI(req),
409-
errorCode: errorCode || null,
410+
errorCode: errorCode === undefined ? null : errorCode,
410411
objectSize: getObjectSize(req, res),
411412
totalTime: calculateTotalTime(params.startTime, params.onFinishEndTime),
412413
turnAroundTime: calculateTurnAroundTime(params.startTurnAroundTime, endTurnAroundTime),
413-
referer: req.headers.referer || null,
414-
userAgent: req.headers['user-agent'] || null,
415-
versionID: req.query ? req.query.versionId || null : null,
414+
referer: req.headers.referer === undefined ? null : req.headers.referer,
415+
userAgent: req.headers['user-agent'] === undefined ? null : req.headers['user-agent'],
416+
versionID: !req.query ? null : req.query.versionId === undefined ? null : req.query.versionId,
416417
signatureVersion: authInfo ? authInfo.getAuthVersion() : null,
417418
cipherSuite: req.socket.encrypted ? req.socket.getCipher()['standardName'] : null,
418419
authenticationType: authInfo ? authInfo.getAuthType() : null,
419-
hostHeader: req.headers.host || null,
420+
hostHeader: req.headers.host === undefined ? null : req.headers.host,
420421
tlsVersion: req.socket.encrypted ? req.socket.getCipher()['version'] : null,
421422
aclRequired: null, // TODO: CLDSRV-774
422423
// hostID: null, // NOT IMPLEMENTED
423424
// accessPointARN: null, // NOT IMPLEMENTED
424425
425426
// Shared between AWS access server logs and Analytics logs
426-
bucketOwner: params.bucketOwner || null,
427-
bucketName: params.bucketName || null, // AWS "Bucket" field
427+
bucketOwner: params.bucketOwner === undefined ? null : params.bucketOwner,
428+
bucketName: params.bucketName === undefined ? null : params.bucketName, // AWS "Bucket" field
428429
// eslint-disable-next-line camelcase
429-
req_id: requestID || null, // AWS "Request ID" field
430+
req_id: requestID === undefined ? null : requestID, // AWS "Request ID" field
430431
bytesSent: getBytesSent(res, bytesSent),
431432
clientIP: getRemoteIPFromRequest(req), // AWS 'Remote IP' field
432-
httpCode: res.statusCode || null, // AWS "HTTP Status" field
433-
objectKey: params.objectKey || null, // AWS "Key" field
433+
httpCode: res.statusCode === undefined ? null : res.statusCode, // AWS "HTTP Status" field
434+
objectKey: params.objectKey === undefined ? null : params.objectKey, // AWS "Key" field
434435
435436
// Scality server access logs extra fields
436437
logFormatVersion: SERVER_ACCESS_LOG_FORMAT_VERSION,
437438
loggingEnabled: params.enabled,
438439
loggingTargetBucket: params.loggingEnabled ? params.loggingEnabled.TargetBucket : null,
439440
loggingTargetPrefix: params.loggingEnabled ? params.loggingEnabled.TargetPrefix : null,
440441
awsAccessKeyID: authInfo ? authInfo.getAccessKey() : null,
441-
raftSessionID: params.raftSessionID || null,
442+
raftSessionID: params.raftSessionID === undefined ? null : params.raftSessionID,
442443
})}\n`);
443444
}
444445

‎schema/server_access_log.schema.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -149,8 +149,8 @@
149149
"type": ["string", "null"]
150150
},
151151
"bucketName": {
152-
"description": "Represents the AWS server access log 'Bucket' field.",
153-
"type": "string"
152+
"description": "Represents the AWS server access log 'Bucket' field. Null for ListBuckets",
153+
"type": ["string", "null"]
154154
},
155155
"req_id": {
156156
"description": "Represents the AWS server access log 'Request ID' field. Matches the req_id of the stdout logs.",

‎tests/unit/utils/serverAccessLogger.js‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -180,10 +180,10 @@ describe('serverAccessLogger utility functions', () => {
180180
assert.strictEqual(result, 'REST.GET.LOGGING_STATUS');
181181
});
182182

183-
it('should return REST.POST.OBJECT for completeMultipartUpload', () => {
183+
it('should return REST.POST.UPLOAD for completeMultipartUpload', () => {
184184
const req = { method: 'POST', apiMethod: 'completeMultipartUpload' };
185185
const result = getOperation(req);
186-
assert.strictEqual(result, 'REST.POST.OBJECT');
186+
assert.strictEqual(result, 'REST.POST.UPLOAD');
187187
});
188188

189189
it('should return REST.method.UNKNOWN for unknown apiMethod', () => {
@@ -328,7 +328,7 @@ describe('serverAccessLogger utility functions', () => {
328328
getHeader: name => name === 'Content-Length' ? '12345' : null,
329329
};
330330
const result = getObjectSize(request, response);
331-
assert.strictEqual(result, '12345');
331+
assert.strictEqual(result, 12345);
332332
});
333333

334334
it('should return Content-Length from request for objectPut', () => {
@@ -340,7 +340,7 @@ describe('serverAccessLogger utility functions', () => {
340340
getHeader: () => null,
341341
};
342342
const result = getObjectSize(request, response);
343-
assert.strictEqual(result, '54321');
343+
assert.strictEqual(result, 54321);
344344
});
345345

346346
it('should return Content-Length from request for objectPutPart', () => {
@@ -352,7 +352,7 @@ describe('serverAccessLogger utility functions', () => {
352352
getHeader: () => null,
353353
};
354354
const result = getObjectSize(request, response);
355-
assert.strictEqual(result, '67890');
355+
assert.strictEqual(result, 67890);
356356
});
357357

358358
it('should handle Content-Length of 0 for objectGet', () => {
@@ -361,7 +361,7 @@ describe('serverAccessLogger utility functions', () => {
361361
getHeader: name => name === 'Content-Length' ? '0' : null,
362362
};
363363
const result = getObjectSize(request, response);
364-
assert.strictEqual(result, '0');
364+
assert.strictEqual(result, 0);
365365
});
366366

367367
it('should handle Content-Length of number 0 for objectGet', () => {
@@ -382,7 +382,7 @@ describe('serverAccessLogger utility functions', () => {
382382
getHeader: () => null,
383383
};
384384
const result = getObjectSize(request, response);
385-
assert.strictEqual(result, '0');
385+
assert.strictEqual(result, 0);
386386
});
387387

388388
it('should handle Content-Length of number 0 for objectPut', () => {
@@ -821,7 +821,7 @@ describe('serverAccessLogger utility functions', () => {
821821
assert.strictEqual(loggedData.operation, 'REST.GET.OBJECT');
822822
assert.strictEqual(loggedData.requestURI, 'GET /test-bucket/test-key.txt HTTP/1.1');
823823
assert.strictEqual(loggedData.errorCode, null);
824-
assert.strictEqual(loggedData.objectSize, '2048');
824+
assert.strictEqual(loggedData.objectSize, 2048);
825825
assert.strictEqual(loggedData.totalTime, '9');
826826
assert.strictEqual(loggedData.turnAroundTime, '1');
827827
assert.strictEqual(loggedData.referer, 'https://example.com');
@@ -992,7 +992,7 @@ describe('serverAccessLogger utility functions', () => {
992992
assert.strictEqual(mockLogger.write.callCount, 1);
993993
const loggedData = JSON.parse(mockLogger.write.firstCall.args[0].trim());
994994
assert.strictEqual(loggedData.operation, 'REST.PUT.OBJECT');
995-
assert.strictEqual(loggedData.objectSize, '5000');
995+
assert.strictEqual(loggedData.objectSize, 5000);
996996
assert.strictEqual(loggedData.bytesReceived, 5000);
997997
assert.strictEqual(loggedData.totalTime, '2');
998998
});

0 commit comments

Comments
 (0)