INTEGRATION [PR#6043 > development/9.3] ✨(CLDSRV-813) update CloudServer XML ListObjectsV2 to support optional attributes

lib/api/bucketGet.js

@@ -138,6 +138,7 @@ function processVersions(bucketName, listParams, list) {
         const objectKey = escapeXmlFn(item.key);
         const isLatest = lastKey !== objectKey;
         lastKey = objectKey;
+
         xml.push(
             v.IsDeleteMarker ? '<DeleteMarker>' : '<Version>',
             `<Key>${objectKey}</Key>`,
@@ -153,14 +154,17 @@ function processVersions(bucketName, listParams, list) {
             `<ID>${v.Owner.ID}</ID>`,
             `<DisplayName>${v.Owner.DisplayName}</DisplayName>`,
             '</Owner>',
+            ...processOptionalAttributes(v, listParams.optionalAttributes),
             `<StorageClass>${v.StorageClass}</StorageClass>`,
             v.IsDeleteMarker ? '</DeleteMarker>' : '</Version>'
         );
     });
+
     list.CommonPrefixes.forEach(item => {
         const val = escapeXmlFn(item);
         xml.push(`<CommonPrefixes><Prefix>${val}</Prefix></CommonPrefixes>`);
     });
+
     xml.push('</ListVersionsResult>');
     return xml.join('');
 }
@@ -224,6 +228,7 @@ function processMasterVersions(bucketName, listParams, list) {
         if (v.isDeleteMarker) {
             return null;
         }
+
         const objectKey = escapeXmlFn(item.key);
         xml.push(
             '<Contents>',
@@ -232,6 +237,7 @@ function processMasterVersions(bucketName, listParams, list) {
             `<ETag>&quot;${v.ETag}&quot;</ETag>`,
             `<Size>${v.Size}</Size>`
         );
+
         if (!listParams.v2 || listParams.fetchOwner) {
             xml.push(
                 '<Owner>',
@@ -240,6 +246,9 @@ function processMasterVersions(bucketName, listParams, list) {
                 '</Owner>'
             );
         }
+
+        xml.push(...processOptionalAttributes(v, listParams.optionalAttributes));
+
         return xml.push(
             `<StorageClass>${v.StorageClass}</StorageClass>`,
             '</Contents>'
@@ -253,20 +262,58 @@ function processMasterVersions(bucketName, listParams, list) {
     return xml.join('');
 }
 
+function processOptionalAttributes(item, optionalAttributes) {
+    const xml = [];
+    const userMetadata = new Set();
+
+    for (const attribute of optionalAttributes) {
+        switch (attribute) {
+            case 'RestoreStatus':
+                xml.push('<RestoreStatus>');
+                xml.push(`<IsRestoreInProgress>${!!item.restoreStatus?.inProgress}</IsRestoreInProgress>`);
+
+                if (item.restoreStatus?.expiryDate) {
+                    xml.push(`<RestoreExpiryDate>${item.restoreStatus?.expiryDate}</RestoreExpiryDate>`);
+                }
+
+                xml.push('</RestoreStatus>');
+                break;
+            case 'x-amz-meta-*':
+                for (const key of Object.keys(item.userMetadata)) {
+                    userMetadata.add(key);
+                }
+                break;
+            default:
+                if (item.userMetadata?.[attribute]) {
+                    userMetadata.add(attribute);
+                }
+        }
+    }
+
+    for (const key of userMetadata) {
+        xml.push(`<${key}>${item.userMetadata[key]}</${key}>`);
+    }
+
+    return xml;
+}
+
 function handleResult(listParams, requestMaxKeys, encoding, authInfo,
     bucketName, list, corsHeaders, log, callback) {
     // eslint-disable-next-line no-param-reassign
     listParams.maxKeys = requestMaxKeys;
     // eslint-disable-next-line no-param-reassign
     listParams.encoding = encoding;
+
     let res;
     if (listParams.listingType === 'DelimiterVersions') {
         res = processVersions(bucketName, listParams, list);
     } else {
         res = processMasterVersions(bucketName, listParams, list);
     }
+
     pushMetric('listBucket', log, { authInfo, bucket: bucketName });
     monitoring.promMetrics('GET', bucketName, '200', 'listBucket');
+
     return callback(null, res, corsHeaders);
 }
 
@@ -286,7 +333,11 @@ function bucketGet(authInfo, request, log, callback) {
     const v2 = params['list-type'];
 
     const optionalAttributes =
-        request.headers['x-amz-optional-object-attributes']?.split(',').map(attr => attr.trim()) ?? [];
+        request.headers['x-amz-optional-object-attributes']
+            ?.split(',')
+            .map(attr => attr.trim())
+            .map(attr => attr !== 'RestoreStatus' ? attr.toLowerCase() : attr)
+        ?? [];
     if (optionalAttributes.some(attr => !attr.startsWith('x-amz-meta-') && attr != 'RestoreStatus')) {
         return callback(
             errorInstances.InvalidArgument.customizeDescription('Invalid attribute name specified')
@@ -344,6 +395,7 @@ function bucketGet(authInfo, request, log, callback) {
         listingType: 'DelimiterMaster',
         maxKeys: actualMaxKeys,
         prefix: params.prefix,
+        optionalAttributes,
     };
 
     if (params.delimiter) {

lib/api/metadataSearch.js

@@ -19,6 +19,8 @@ function handleResult(listParams, requestMaxKeys, encoding, authInfo,
     listParams.maxKeys = requestMaxKeys;
     // eslint-disable-next-line no-param-reassign
     listParams.encoding = encoding;
+    // eslint-disable-next-line no-param-reassign
+    listParams.optionalAttributes = [];
     let res;
     if (listParams.listingType === 'DelimiterVersions') {
         res = processVersions(bucketName, listParams, list);

lib/routes/veeam/list.js

@@ -25,6 +25,7 @@ function buildXMLResponse(request, arrayOfFiles, versioned = false) {
         prefix: validPath,
         maxKeys: parsedQs['max-keys'] || 1000,
         delimiter: '/',
+        optionalAttributes: [],
     };
     const list = {
         IsTruncated: false,

tests/functional/aws-node-sdk/test/bucket/get.js

@@ -4,14 +4,32 @@ const {
     PutObjectCommand,
     ListObjectsCommand,
     ListObjectsV2Command,
+    S3Client,
 } = require('@aws-sdk/client-s3');
+const { streamCollector } = require('@smithy/node-http-handler');
+const {
+    IAMClient,
+    CreatePolicyCommand,
+    CreateUserCommand,
+    AttachUserPolicyCommand,
+    CreateAccessKeyCommand,
+    DetachUserPolicyCommand,
+    DeletePolicyCommand,
+    DeleteUserCommand,
+} = require('@aws-sdk/client-iam');
+const { parseStringPromise } = require('xml2js');
 
 const withV4 = require('../support/withV4');
 const BucketUtility = require('../../lib/utility/bucket-util');
 const bucketSchema = require('../../schema/bucket');
 const bucketSchemaV2 = require('../../schema/bucketV2');
-const { generateToken, decryptToken } =
-    require('../../../../../lib/api/apiUtils/object/continueToken');
+const { generateToken, decryptToken } = require('../../../../../lib/api/apiUtils/object/continueToken');
+const getConfig = require('../support/config');
+const { config } = require('../../../../../lib/Config');
+
+const isVaultScality = config.backends.auth !== 'mem';
+const internalPortBypassBP = config.internalPort;
+const vaultHost = config.vaultd?.host || 'localhost';
 
 const tests = [
     {
@@ -535,5 +553,207 @@ describe('GET Bucket - AWS.S3.listObjects', () => {
                 assert.strictEqual($metadata.httpStatusCode, 200);
             });
         });
+
+        const describeBypass = isVaultScality && internalPortBypassBP ? describe : describe.skip;
+        describeBypass('x-amz-optional-attributes header', () => {
+            let policyWithoutPermission;
+            let userWithoutPermission;
+            let s3ClientWithoutPermission;
+
+            const iamConfig = getConfig('default', { region: 'us-east-1' });
+            iamConfig.endpoint = `http://${vaultHost}:8600`;
+            const iamClient = new IAMClient(iamConfig);
+
+            before(async () => {
+                const policyRes = await iamClient.send(new CreatePolicyCommand({
+                    PolicyName: 'bp-bypass-policy',
+                    PolicyDocument: JSON.stringify({
+                        Version: '2012-10-17',
+                        Statement: [{
+                            Sid: 'AllowS3ListBucket',
+                            Effect: 'Allow',
+                            Action: [
+                                's3:ListBucket',
+                            ],
+                            Resource: ['*'],
+                        }],
+                    }),
+                }));
+                policyWithoutPermission = policyRes.Policy;
+                const userRes = await iamClient.send(new CreateUserCommand({ UserName: 'user-without-permission' }));
+                userWithoutPermission = userRes.User;
+                await iamClient.send(new AttachUserPolicyCommand({
+                    UserName: userWithoutPermission.UserName,
+                    PolicyArn: policyWithoutPermission.Arn,
+                }));
+
+                const accessKeyRes = await iamClient.send(new CreateAccessKeyCommand({
+                    UserName: userWithoutPermission.UserName,
+                }));
+                const accessKey = accessKeyRes.AccessKey;
+                const s3Config = getConfig('default', {
+                    credentials: {
+                        accessKeyId: accessKey.AccessKeyId,
+                        secretAccessKey: accessKey.SecretAccessKey,
+                    },
+                });
+                s3ClientWithoutPermission = new S3Client(s3Config);
+            });
+
+            after(async () => {
+                await iamClient.send(new DetachUserPolicyCommand({
+                    UserName: userWithoutPermission.UserName,
+                    PolicyArn: policyWithoutPermission.Arn,
+                }));
+                await iamClient.send(new DeletePolicyCommand({ PolicyArn: policyWithoutPermission.Arn }));
+                await iamClient.send(new DeleteUserCommand({ UserName: userWithoutPermission.UserName }));
+            });
+
+            const listObjectsV2WithOptionalAttributes = async (s3, bucket, headerValue) => {
+                const localS3 = s3;
+                let rawXml = '';
+
+                const addHeaderMiddleware = next => async args => {
+                    const localArgs = args;
+                    localArgs.request.headers['x-amz-optional-object-attributes'] = headerValue;
+                    return next(localArgs);
+                };
+
+                const originalHandler = s3.config.requestHandler;
+                const wrappedHandler = {
+                    async handle(request, options) {
+                        const { response } = await originalHandler.handle(request, options);
+
+                        if (response && response.body) {
+                            const collected = await streamCollector(response.body);
+                            const buffer = Buffer.from(collected);
+                            rawXml = buffer.toString('utf-8');
+
+                            const { Readable } = require('stream');
+                            response.body = Readable.from([buffer]);
+                        }
+
+                        return { response };
+                    },
+                    destroy() {
+                        if (originalHandler.destroy) {
+                            originalHandler.destroy();
+                        }
+                    }
+                };
+
+                localS3.config.requestHandler = wrappedHandler;
+                localS3.middlewareStack.add(addHeaderMiddleware, {
+                    step: 'build',
+                    name: 'addOptionalAttributesHeader',
+                });
+
+                try {
+                    const result = await s3.send(new ListObjectsV2Command({ Bucket: bucket }));
+
+                    if (!rawXml) {
+                        return result;
+                    }
+
+                    const parsedXml = await parseStringPromise(rawXml);
+                    const contents = result.Contents;
+                    const parsedContents = parsedXml?.ListBucketResult?.Contents;
+
+                    if (!contents || !parsedContents) {
+                        return result;
+                    }
+
+                    if (parsedContents[0]?.['x-amz-meta-department']) {
+                        contents[0]['x-amz-meta-department'] = parsedContents[0]['x-amz-meta-department'][0];
+                    }
+
+                    if (parsedContents[0]?.['x-amz-meta-hr']) {
+                        contents[0]['x-amz-meta-hr'] = parsedContents[0]['x-amz-meta-hr'][0];
+                    }
+
+                    return result;
+                } finally {
+                    localS3.config.requestHandler = originalHandler;
+                    localS3.middlewareStack.remove('addOptionalAttributesHeader');
+                }
+            };
+
+            it('should return an XML if the header is set', async () => {
+                const s3 = bucketUtil.s3;
+                const Bucket = bucketName;
+
+                await s3.send(new PutObjectCommand({
+                    Bucket,
+                    Key: 'super-power-object',
+                    Metadata: {
+                        department: 'sales',
+                        hr: 'true',
+                    },
+                }));
+                const result = await listObjectsV2WithOptionalAttributes(
+                    s3,
+                    Bucket,
+                    'x-amz-meta-*,RestoreStatus,x-amz-meta-department',
+                );
+
+                assert.strictEqual(result.Contents.length, 1);
+                assert.strictEqual(result.Contents[0].Key, 'super-power-object');
+                assert.strictEqual(result.Contents[0]['x-amz-meta-department'], 'sales');
+                assert.strictEqual(result.Contents[0]['x-amz-meta-hr'], 'true');
+            });
+
+            it('should reject the request if the user does not have the permission', async () => {
+                const s3 = bucketUtil.s3;
+                const Bucket = bucketName;
+
+                await s3.send(new PutObjectCommand({
+                    Bucket,
+                    Key: 'super-power-object',
+                    Metadata: {
+                        department: 'sales',
+                        hr: 'true',
+                    },
+                }));
+
+                try {
+                    await listObjectsV2WithOptionalAttributes(
+                        s3ClientWithoutPermission,
+                        Bucket,
+                        'x-amz-meta-*,RestoreStatus,x-amz-meta-department',
+                    );
+                    throw new Error('Request should have been rejected');
+                } catch (err) {
+                    if (err.message === 'Request should have been rejected') {
+                        throw err;
+                    }
+                    assert.strictEqual(err.$metadata.httpStatusCode, 403);
+                    assert.strictEqual(err.name, 'AccessDenied');
+                }
+            });
+
+            it('should always (ignore permission) return an XML when the header is RestoreStatus', async () => {
+                const s3 = bucketUtil.s3;
+                const Bucket = bucketName;
+
+                await s3.send(new PutObjectCommand({
+                    Bucket,
+                    Key: 'super-power-object',
+                    Metadata: {
+                        department: 'sales',
+                        hr: 'true',
+                    },
+                }));
+                const result = await listObjectsV2WithOptionalAttributes(
+                    s3ClientWithoutPermission,
+                    Bucket,
+                    'RestoreStatus',
+                );
+
+                assert.strictEqual(result.Contents.length, 1);
+                assert.strictEqual(result.Contents[0].Key, 'super-power-object');
+                assert.strictEqual(result.Contents[0]['x-amz-meta-department'], undefined);
+                assert.strictEqual(result.Contents[0]['x-amz-meta-hr'], undefined);
+            });
+        });
     });
 });