From 29d2d54b15ec840bdc71454f87927052a3258d7e Mon Sep 17 00:00:00 2001 From: Mickael Bourgois Date: Fri, 18 Jul 2025 14:55:28 +0200 Subject: [PATCH 01/10] CLDSRV-720: Fix mocha config to run ft tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/mochajs/mocha/blob/main/CHANGELOG.md#boom-breaking-changes-2 > #4175: Having been deprecated with a warning since v7.0.0, mocha.opts is no longer supported (@juergba) > > ✨ WORKAROUND: Replace mocha.opts with a configuration file. --- package.json | 4 ++++ tests/functional/aws-node-sdk/test/mocha.opts | 3 --- tests/functional/raw-node/package.json | 6 +++++- tests/functional/raw-node/test/mocha.opts | 3 --- 4 files changed, 9 insertions(+), 7 deletions(-) delete mode 100644 tests/functional/aws-node-sdk/test/mocha.opts delete mode 100644 tests/functional/raw-node/test/mocha.opts diff --git a/package.json b/package.json index a8351f3cb9..ca065c2aca 100644 --- a/package.json +++ b/package.json @@ -76,6 +76,10 @@ "jsonwebtoken": "^9.0.0", "nan": "v2.22.0" }, + "mocha": { + "recursive": true, + "timeout": 40000 + }, "scripts": { "cloudserver": "S3METADATA=mongodb npm-run-all --parallel start_dataserver start_s3server", "ft_awssdk": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/ --exit", diff --git a/tests/functional/aws-node-sdk/test/mocha.opts b/tests/functional/aws-node-sdk/test/mocha.opts deleted file mode 100644 index 4e0126ebb4..0000000000 --- a/tests/functional/aws-node-sdk/test/mocha.opts +++ /dev/null @@ -1,3 +0,0 @@ ---recursive ---timeout 40000 ---ui tdd \ No newline at end of file diff --git a/tests/functional/raw-node/package.json b/tests/functional/raw-node/package.json index dbebc265c3..df246584b6 100644 --- a/tests/functional/raw-node/package.json +++ b/tests/functional/raw-node/package.json @@ -14,5 +14,9 @@ "test": "mocha -t 40000 test/ --exit", "test-debug": "_mocha -t 40000 test/ --exit" }, - "author": "" + "author": "", + "mocha": { + "recursive": true, + "timeout": 40000 + } } diff --git a/tests/functional/raw-node/test/mocha.opts b/tests/functional/raw-node/test/mocha.opts deleted file mode 100644 index e3c79bad31..0000000000 --- a/tests/functional/raw-node/test/mocha.opts +++ /dev/null @@ -1,3 +0,0 @@ ---recursive ---timeout 40000 ---ui tdd From a20d248d7662d0da625d80791dc6fa08dd93173b Mon Sep 17 00:00:00 2001 From: Mickael Bourgois Date: Fri, 18 Jul 2025 14:57:34 +0200 Subject: [PATCH 02/10] CLDSRV-720: Fix import path in tests --- tests/functional/aws-node-sdk/test/object/objectCopy.js | 2 +- tests/functional/aws-node-sdk/test/versioning/objectCopy.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/functional/aws-node-sdk/test/object/objectCopy.js b/tests/functional/aws-node-sdk/test/object/objectCopy.js index c56a24682c..39cda4b237 100644 --- a/tests/functional/aws-node-sdk/test/object/objectCopy.js +++ b/tests/functional/aws-node-sdk/test/object/objectCopy.js @@ -7,7 +7,7 @@ const { fakeMetadataTransition, fakeMetadataArchive } = require('../utils/init') const { taggingTests } = require('../../lib/utility/tagging'); const genMaxSizeMetaHeaders = require('../../lib/utility/genMaxSizeMetaHeaders'); -const constants = require('../../../../constants'); +const constants = require('../../../../../constants'); const sourceBucketName = 'supersourcebucket8102016'; const sourceObjName = 'supersourceobject'; diff --git a/tests/functional/aws-node-sdk/test/versioning/objectCopy.js b/tests/functional/aws-node-sdk/test/versioning/objectCopy.js index 752778afac..73a94a32fd 100644 --- a/tests/functional/aws-node-sdk/test/versioning/objectCopy.js +++ b/tests/functional/aws-node-sdk/test/versioning/objectCopy.js @@ -7,7 +7,7 @@ const { removeAllVersions } = require('../../lib/utility/versioning-util'); const customS3Request = require('../../lib/utility/customS3Request'); const { taggingTests } = require('../../lib/utility/tagging'); -const constants = require('../../lib/constants'); +const constants = require('../../../../../constants'); const sourceBucketName = 'supersourcebucket8102016'; const sourceObjName = 'supersourceobject'; From ccbb77438e1f6532d6d6d773add68ebf4d5c9642 Mon Sep 17 00:00:00 2001 From: Mickael Bourgois Date: Fri, 18 Jul 2025 16:03:54 +0200 Subject: [PATCH 03/10] CLDSRV-720: Fix replace bluebird with Promise cf: CLDSRV-553 --- .../aws-node-sdk/test/bucket/get.js | 204 +++++++----------- .../aws-node-sdk/test/bucket/put.js | 6 +- .../aws-node-sdk/test/object/putObjTagging.js | 2 +- 3 files changed, 84 insertions(+), 128 deletions(-) diff --git a/tests/functional/aws-node-sdk/test/bucket/get.js b/tests/functional/aws-node-sdk/test/bucket/get.js index af78eb9a20..0267ad7a65 100644 --- a/tests/functional/aws-node-sdk/test/bucket/get.js +++ b/tests/functional/aws-node-sdk/test/bucket/get.js @@ -337,7 +337,7 @@ describe('GET Bucket - AWS.S3.listObjects', () => { }); afterEach(done => { - bucketUtil.empty(bucketName).catch(done).done(() => done()); + bucketUtil.empty(bucketName).then(() => done()).catch(done); }); tests.forEach(test => { @@ -357,181 +357,137 @@ describe('GET Bucket - AWS.S3.listObjects', () => { }); tests.forEach(test => { - it(`v2 should ${test.name}`, done => { + it(`v2 should ${test.name}`, async () => { const s3 = bucketUtil.s3; const Bucket = bucketName; - Promise - .mapSeries(test.objectPutParams(Bucket), - param => s3.putObject(param).promise()) - .then(() => - s3.listObjectsV2(test.listObjectParams(Bucket)) - .promise()) - .then(data => { - const isValidResponse = - tv4.validate(data, bucketSchemaV2); - if (!isValidResponse) { - throw new Error(tv4.error); - } - return data; - }).then(data => { - test.assertions(data, Bucket); - done(); - }) - .catch(done); + for (const param of test.objectPutParams(Bucket)) { + await s3.putObject(param).promise(); + } + const data = await s3.listObjectsV2(test.listObjectParams(Bucket)).promise(); + const isValidResponse = tv4.validate(data, bucketSchemaV2); + if (!isValidResponse) { + throw new Error(tv4.error); + } + test.assertions(data, Bucket); }); }); ['&', '"quot', '\'apos', 'gt'].forEach(k => { - it(`should list objects with key ${k} as Prefix`, done => { + it(`should list objects with key ${k} as Prefix`, async () => { const s3 = bucketUtil.s3; const Bucket = bucketName; const objects = [{ Bucket, Key: k }]; - Promise - .mapSeries(objects, param => s3.putObject(param).promise()) - .then(() => s3.listObjects({ Bucket, Prefix: k }).promise()) - .then(data => { - const isValidResponse = tv4.validate(data, - bucketSchema); - if (!isValidResponse) { - throw new Error(tv4.error); - } - return data; - }).then(data => { - assert.deepStrictEqual(data.Prefix, k); - done(); - }) - .catch(done); + for (const param of objects) { + await s3.putObject(param).promise(); + } + const data = await s3.listObjects({ Bucket, Prefix: k }).promise(); + const isValidResponse = tv4.validate(data, bucketSchema); + if (!isValidResponse) { + throw new Error(tv4.error); + } + assert.deepStrictEqual(data.Prefix, k); }); }); ['&', '"quot', '\'apos', 'gt'].forEach(k => { - it(`should list objects with key ${k} as Marker`, done => { + it(`should list objects with key ${k} as Marker`, async () => { const s3 = bucketUtil.s3; const Bucket = bucketName; const objects = [{ Bucket, Key: k }]; - Promise - .mapSeries(objects, param => s3.putObject(param).promise()) - .then(() => s3.listObjects({ Bucket, Marker: k }).promise()) - .then(data => { - const isValidResponse = tv4.validate(data, - bucketSchema); - if (!isValidResponse) { - throw new Error(tv4.error); - } - return data; - }).then(data => { - assert.deepStrictEqual(data.Marker, k); - done(); - }) - .catch(done); + for (const param of objects) { + await s3.putObject(param).promise(); + } + const data = await s3.listObjects({ Bucket, Marker: k }).promise(); + const isValidResponse = tv4.validate(data, bucketSchema); + if (!isValidResponse) { + throw new Error(tv4.error); + } + assert.deepStrictEqual(data.Marker, k); }); }); ['&', '"quot', '\'apos', 'gt'].forEach(k => { - it(`should list objects with key ${k} as NextMarker`, done => { + it(`should list objects with key ${k} as NextMarker`, async () => { const s3 = bucketUtil.s3; const Bucket = bucketName; const objects = [{ Bucket, Key: k }, { Bucket, Key: 'zzz' }]; - Promise - .mapSeries(objects, param => s3.putObject(param).promise()) - .then(() => s3.listObjects({ Bucket, MaxKeys: 1, - Delimiter: 'foo' }).promise()) - .then(data => { - const isValidResponse = tv4.validate(data, - bucketSchema); - if (!isValidResponse) { - throw new Error(tv4.error); - } - return data; - }).then(data => { - assert.strictEqual(data.NextMarker, k); - done(); - }) - .catch(done); + for (const param of objects) { + await s3.putObject(param).promise(); + } + const data = await s3.listObjects({ Bucket, MaxKeys: 1, + Delimiter: 'foo' }).promise(); + const isValidResponse = tv4.validate(data, bucketSchema); + if (!isValidResponse) { + throw new Error(tv4.error); + } + assert.strictEqual(data.NextMarker, k); }); }); ['&', '"quot', '\'apos', 'gt'].forEach(k => { - it(`should list objects with key ${k} as StartAfter`, done => { + it(`should list objects with key ${k} as StartAfter`, async () => { const s3 = bucketUtil.s3; const Bucket = bucketName; const objects = [{ Bucket, Key: k }]; - Promise - .mapSeries(objects, param => s3.putObject(param).promise()) - .then(() => s3.listObjectsV2( - { Bucket, StartAfter: k }).promise()) - .then(data => { - const isValidResponse = tv4.validate(data, - bucketSchemaV2); - if (!isValidResponse) { - throw new Error(tv4.error); - } - return data; - }).then(data => { - assert.deepStrictEqual(data.StartAfter, k); - done(); - }) - .catch(done); + for (const param of objects) { + await s3.putObject(param).promise(); + } + const data = await s3.listObjectsV2( + { Bucket, StartAfter: k }).promise(); + const isValidResponse = tv4.validate(data, bucketSchemaV2); + if (!isValidResponse) { + throw new Error(tv4.error); + } + assert.deepStrictEqual(data.StartAfter, k); }); }); ['&', '"quot', '\'apos', 'gt'].forEach(k => { it(`should list objects with key ${k} as ContinuationToken`, - done => { + async () => { const s3 = bucketUtil.s3; const Bucket = bucketName; const objects = [{ Bucket, Key: k }]; - Promise - .mapSeries(objects, param => s3.putObject(param).promise()) - .then(() => s3.listObjectsV2({ - Bucket, - ContinuationToken: generateToken(k), - }).promise()) - .then(data => { - const isValidResponse = tv4.validate(data, - bucketSchemaV2); - if (!isValidResponse) { - throw new Error(tv4.error); - } - return data; - }).then(data => { - assert.deepStrictEqual( - decryptToken(data.ContinuationToken), k); - done(); - }) - .catch(done); + for (const param of objects) { + await s3.putObject(param).promise(); + } + const data = await s3.listObjectsV2({ + Bucket, + ContinuationToken: generateToken(k), + }).promise(); + const isValidResponse = tv4.validate(data, bucketSchemaV2); + if (!isValidResponse) { + throw new Error(tv4.error); + } + assert.deepStrictEqual( + decryptToken(data.ContinuationToken), k); }); }); ['&', '"quot', '\'apos', 'gt'].forEach(k => { it(`should list objects with key ${k} as NextContinuationToken`, - done => { + async () => { const s3 = bucketUtil.s3; const Bucket = bucketName; const objects = [{ Bucket, Key: k }, { Bucket, Key: 'zzz' }]; - Promise - .mapSeries(objects, param => s3.putObject(param).promise()) - .then(() => s3.listObjectsV2({ Bucket, MaxKeys: 1, - Delimiter: 'foo' }).promise()) - .then(data => { - const isValidResponse = tv4.validate(data, - bucketSchemaV2); - if (!isValidResponse) { - throw new Error(tv4.error); - } - return data; - }).then(data => { - assert.strictEqual( - decryptToken(data.NextContinuationToken), k); - done(); - }) - .catch(done); + + for (const param of objects) { + await s3.putObject(param).promise(); + } + const data = await s3.listObjectsV2({ Bucket, MaxKeys: 1, + Delimiter: 'foo' }).promise(); + const isValidResponse = tv4.validate(data, bucketSchemaV2); + if (!isValidResponse) { + throw new Error(tv4.error); + } + assert.strictEqual( + decryptToken(data.NextContinuationToken), k); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/bucket/put.js b/tests/functional/aws-node-sdk/test/bucket/put.js index 6bf63335f2..43b994a7b5 100644 --- a/tests/functional/aws-node-sdk/test/bucket/put.js +++ b/tests/functional/aws-node-sdk/test/bucket/put.js @@ -257,9 +257,9 @@ describe('PUT Bucket - AWS.S3.createBucket', () => { location => { describeSkipAWS(`bucket creation with location: ${location}`, () => { - after(done => - bucketUtil.deleteOne(bucketName) - .then(() => done()).catch(() => done())); + after(done => { + bucketUtil.deleteOne(bucketName).finally(done); + }); it(`should create bucket with location: ${location}`, done => { bucketUtil.s3.createBucket( { diff --git a/tests/functional/aws-node-sdk/test/object/putObjTagging.js b/tests/functional/aws-node-sdk/test/object/putObjTagging.js index 9912c9ff2c..bc6a4f93e9 100644 --- a/tests/functional/aws-node-sdk/test/object/putObjTagging.js +++ b/tests/functional/aws-node-sdk/test/object/putObjTagging.js @@ -55,7 +55,7 @@ describe('PUT object taggings', () => { return s3.putObject({ Bucket: bucketName, Key: objectName }, done); })); - afterEach(() => { + afterEach(async () => { process.stdout.write('Emptying bucket'); return bucketUtil.empty(bucketName) .then(() => { From 479edda41f1663c8df5615d5b94d338077252345 Mon Sep 17 00:00:00 2001 From: Mickael Bourgois Date: Fri, 18 Jul 2025 16:04:44 +0200 Subject: [PATCH 04/10] CLDSRV-720: Fix socket reused when counting bytes --- tests/functional/aws-node-sdk/test/object/100-continue.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/functional/aws-node-sdk/test/object/100-continue.js b/tests/functional/aws-node-sdk/test/object/100-continue.js index db76fb7f6b..855d2744ec 100644 --- a/tests/functional/aws-node-sdk/test/object/100-continue.js +++ b/tests/functional/aws-node-sdk/test/object/100-continue.js @@ -35,6 +35,8 @@ class ContinueRequestHandler { getRequestOptions() { return { + // Prevent socket reuse as a test checks for socket.bytesWritten + agent: new http.Agent({ keepAlive: false }), path: this.path, hostname, port, From c447d70f8247264ffc3a344640ddaa28d63ba16e Mon Sep 17 00:00:00 2001 From: Mickael Bourgois Date: Fri, 18 Jul 2025 20:06:05 +0200 Subject: [PATCH 05/10] CLDSRV-720: Fix test HEAD and partNumber (of MPU) cf: CLDSRV-638 --- .../aws-node-sdk/test/object/getPartSize.js | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tests/functional/aws-node-sdk/test/object/getPartSize.js b/tests/functional/aws-node-sdk/test/object/getPartSize.js index 32e7040f4e..815bad5c3e 100644 --- a/tests/functional/aws-node-sdk/test/object/getPartSize.js +++ b/tests/functional/aws-node-sdk/test/object/getPartSize.js @@ -10,6 +10,7 @@ const object = 'mpu-test-object'; const emptyObject = 'empty-object'; const nonMpuObject = 'simple-object'; +/** 5MiB */ const bodySize = 1024 * 1024 * 5; const bodyContent = 'a'; const howManyParts = 3; @@ -18,6 +19,13 @@ const invalidPartNumbers = [-1, 0, maximumAllowedPartCount + 1]; let ETags = []; +// Because HEAD has no body, the SDK (v2) returns a generic code such as: +// 400 BadRequest +// 403 Forbidden +// 404 NotFound +// ... +// It will fall back to HTTP statusCode +// Example: 416 InvalidRange will be 416 416 function checkError(err, statusCode, code) { assert.strictEqual(err.statusCode, statusCode); assert.strictEqual(err.code, code); @@ -174,7 +182,7 @@ describe('Part size tests with object head', () => { it('should return an error when requesting part 2 of empty object', done => { headObject({ Key: emptyObject, PartNumber: 2 }, (err, data) => { - checkError(err, 416, 'InvalidRange'); + checkError(err, 416, 416); assert.strictEqual(data, null); done(); }); @@ -183,15 +191,15 @@ describe('Part size tests with object head', () => { it('should return content-length requesting part 1 of non-MPU object', done => { headObject({ Key: nonMpuObject, PartNumber: 1 }, (err, data) => { checkNoError(err); - assert.strictEqual(data.ContentLength, 0); + assert.strictEqual(data.ContentLength, bodySize); done(); }); }); it('should return an error when requesting part 2 of non-MPU object', done => { - headObject({ Key: nonMpuObject, PartNumber: 1 }, (err, data) => { - checkError(err, 416, 'InvalidRange'); - assert.strictEqual(data.ContentLength, bodySize); + headObject({ Key: nonMpuObject, PartNumber: 2 }, (err, data) => { + checkError(err, 416, 416); + assert.strictEqual(data, null); done(); }); }); From 26e0396b60a44aff512d0226eac5a6cd98ae0197 Mon Sep 17 00:00:00 2001 From: Mickael Bourgois Date: Fri, 18 Jul 2025 20:08:37 +0200 Subject: [PATCH 06/10] CLDSRV-720: Try to clean all buckets before list Test can fail if previous tests did not clean buckets --- .../aws-node-sdk/test/service/get.js | 73 +++++++++++++++---- 1 file changed, 57 insertions(+), 16 deletions(-) diff --git a/tests/functional/aws-node-sdk/test/service/get.js b/tests/functional/aws-node-sdk/test/service/get.js index f168f9148c..01b2a5e051 100644 --- a/tests/functional/aws-node-sdk/test/service/get.js +++ b/tests/functional/aws-node-sdk/test/service/get.js @@ -12,6 +12,41 @@ const describeFn = process.env.AWS_ON_AIR ? describe.skip : describe; +async function cleanBucket(bucketUtils, s3, Bucket) { + try { + await bucketUtils.empty(Bucket); + await s3.deleteBucket({ Bucket }).promise(); + } catch (err) { + process.stdout + .write(`Error emptying and deleting bucket: ${err}\n`); + // ignore the error and continue + } +} + +async function cleanAllBuckets(bucketUtils, s3) { + let listingLoop = true; + let ContinuationToken; + + process.stdout.write('Try cleaning all buckets before running the test\n'); + + while (listingLoop) { + const list = await s3.listBuckets({ ContinuationToken }).promise(); + ContinuationToken = list.ContinuationToken; + listingLoop = !!ContinuationToken; + + if (list.Buckets.length) { + process.stdout + .write(`Found ${list.Buckets.length} buckets to clean:\n${ + JSON.stringify(list.Buckets, null, 2)}\n`); + } + + // clean sequentially to avoid overloading + for (const bucket of list.Buckets) { + await cleanBucket(bucketUtils, s3, bucket.Name); + } + } +} + describeFn('GET Service - AWS.S3.listBuckets', function getService() { this.timeout(600000); @@ -98,22 +133,28 @@ describeFn('GET Service - AWS.S3.listBuckets', function getService() { s3 = bucketUtil.s3; s3.config.update({ maxRetries: 0 }); s3.config.update({ httpOptions: { timeout: 0 } }); - async.eachLimit(createdBuckets, 10, (bucketName, moveOn) => { - s3.createBucket({ Bucket: bucketName }, err => { - if (bucketName.endsWith('000')) { - // log to keep ci alive - process.stdout - .write(`creating bucket: ${bucketName}\n`); - } - moveOn(err); - }); - }, - err => { - if (err) { - process.stdout.write(`err creating buckets: ${err}`); - } - done(err); - }); + async.series([ + // if other tests failed to delete their buckets, listings might be wrong + // try toclean all buckets before running the test + next => cleanAllBuckets(bucketUtil, s3).then(next).catch(next), + next => + async.eachLimit(createdBuckets, 10, (bucketName, moveOn) => { + s3.createBucket({ Bucket: bucketName }, err => { + if (bucketName.endsWith('000')) { + // log to keep ci alive + process.stdout + .write(`creating bucket: ${bucketName}\n`); + } + moveOn(err); + }); + }, + err => { + if (err) { + process.stdout.write(`err creating buckets: ${err}`); + } + next(err); + }) + ], done); }); after(done => { From f4346e0f10d9271b2499286c1aec80805430ee26 Mon Sep 17 00:00:00 2001 From: Mickael Bourgois Date: Fri, 18 Jul 2025 21:02:49 +0200 Subject: [PATCH 07/10] CLDSRV-720: Fix test hook metadata mongoclient MongoClient setup returns a promise --- .../functional/aws-node-sdk/test/bucket/getBucketEncryption.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/aws-node-sdk/test/bucket/getBucketEncryption.js b/tests/functional/aws-node-sdk/test/bucket/getBucketEncryption.js index 5b9794f6b2..5b9e498204 100644 --- a/tests/functional/aws-node-sdk/test/bucket/getBucketEncryption.js +++ b/tests/functional/aws-node-sdk/test/bucket/getBucketEncryption.js @@ -25,7 +25,7 @@ describe('aws-sdk test get bucket encryption', () => { before(done => { const config = getConfig('default', { signatureVersion: 'v4' }); s3 = new S3(config); - return metadata.setup(done); + metadata.setup(done); }); beforeEach(done => s3.createBucket({ Bucket: bucketName }, done)); From 4ed1bc2e5e8ab2da0bb886622bc26fb70e475404 Mon Sep 17 00:00:00 2001 From: Mickael Bourgois Date: Sun, 20 Jul 2025 19:36:46 +0200 Subject: [PATCH 08/10] CLDSRV-720: Bump arsenal to fix GCP createMPU --- package.json | 2 +- yarn.lock | 54 +++++++++++++++++++++++++++++++++++++--------------- 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/package.json b/package.json index ca065c2aca..68c96d87eb 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "dependencies": { "@azure/storage-blob": "^12.25.0", "@hapi/joi": "^17.1.1", - "arsenal": "git+https://github.com/scality/Arsenal#8.2.26", + "arsenal": "git+https://github.com/scality/Arsenal#8.2.27", "async": "2.6.4", "aws-sdk": "^2.1692.0", "bucketclient": "scality/bucketclient#8.2.4", diff --git a/yarn.lock b/yarn.lock index b0fc2f6f58..8224604fb1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -229,10 +229,10 @@ resolved "https://registry.yarnpkg.com/@azure/msal-common/-/msal-common-15.3.0.tgz#cc75760f7929588b261b970a1dd1d292d0efdbc8" integrity sha512-lh+eZfibGwtQxFnx+mj6cYWn0pwA8tDnn8CBs9P21nC7Uw5YWRwfXaXdVQSMENZ5ojRqR+NzRaucEo4qUvs3pA== -"@azure/msal-common@15.7.1": - version "15.7.1" - resolved "https://registry.yarnpkg.com/@azure/msal-common/-/msal-common-15.7.1.tgz#6f8df5ace9c94d570b5eec1abc6ebdd4e12831f1" - integrity sha512-a0eowoYfRfKZEjbiCoA5bPT3IlWRAdGSvi63OU23Hv+X6EI8gbvXCoeqokUceFMoT9NfRUWTJSx5FiuzruqT8g== +"@azure/msal-common@15.8.1": + version "15.8.1" + resolved "https://registry.yarnpkg.com/@azure/msal-common/-/msal-common-15.8.1.tgz#2710f0ff9e4b1347c84da1c26857213c7530d76c" + integrity sha512-ltIlFK5VxeJ5BurE25OsJIfcx1Q3H/IZg2LjV9d4vmH+5t4c1UCyRQ/HgKLgXuCZShs7qfc/TC95GYZfsUsJUQ== "@azure/msal-node@^3.2.3": version "3.4.0" @@ -244,11 +244,11 @@ uuid "^8.3.0" "@azure/msal-node@^3.5.0": - version "3.6.1" - resolved "https://registry.yarnpkg.com/@azure/msal-node/-/msal-node-3.6.1.tgz#1f288c3de15bdd734cc41a39bbd50f5a83fd64a1" - integrity sha512-ctcVz4xS+st5KxOlQqgpvA+uDFAa59CvkmumnuhlD2XmNczloKBdCiMQG7/TigSlaeHe01qoOlDjz3TyUAmKUg== + version "3.6.3" + resolved "https://registry.yarnpkg.com/@azure/msal-node/-/msal-node-3.6.3.tgz#d07cee404d071850a4657e86e07b43c1734f97f8" + integrity sha512-95wjsKGyUcAd5tFmQBo5Ug/kOj+hFh/8FsXuxluEvdfbgg6xCimhSP9qnyq6+xIg78/jREkBD1/BSqd7NIDDYQ== dependencies: - "@azure/msal-common" "15.7.1" + "@azure/msal-common" "15.8.1" jsonwebtoken "^9.0.0" uuid "^8.3.0" @@ -1296,9 +1296,9 @@ arraybuffer.prototype.slice@^1.0.4: get-intrinsic "^1.2.6" is-array-buffer "^3.0.4" -"arsenal@git+https://github.com/scality/Arsenal#8.2.26": - version "8.2.26" - resolved "git+https://github.com/scality/Arsenal#80bb045e77ed9a5205af105820dc23b8351d3258" +"arsenal@git+https://github.com/scality/Arsenal#8.2.27": + version "8.2.27" + resolved "git+https://github.com/scality/Arsenal#55b442593c1ecdae69529084c43549104732fe21" dependencies: "@azure/identity" "^4.10.2" "@azure/storage-blob" "^12.27.0" @@ -6339,7 +6339,16 @@ stream-to-pull-stream@^1.7.1: looper "^3.0.0" pull-stream "^3.2.3" -"string-width-cjs@npm:string-width@^4.2.0", string-width@4.2.3, string-width@^4.1.0, string-width@^4.2.0, string-width@^5.0.1, string-width@^5.1.2: +"string-width-cjs@npm:string-width@^4.2.0": + version "4.2.3" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + +string-width@4.2.3, string-width@^4.1.0, string-width@^4.2.0, string-width@^5.0.1, string-width@^5.1.2: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -6402,7 +6411,14 @@ string_decoder@~0.10.x: resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94" integrity sha512-ev2QzSzWPYmy9GuqfIVildA4OdcGLeFZQrq5ys6RtiuF+RQQiZWr8TZNyAcuVXyQRYfEO+MsoB/1BuQVhOJuoQ== -"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: +"strip-ansi-cjs@npm:strip-ansi@^6.0.1": + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== + dependencies: + ansi-regex "^5.0.1" + +strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -7114,8 +7130,7 @@ workerpool@^6.5.1: resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-6.5.1.tgz#060f73b39d0caf97c6db64da004cd01b4c099544" integrity sha512-Fs4dNYcsdpYSAfVxhnl1L5zTksjvOJxtC5hzMNl+1t9B8hTJTdKDyZ5ju7ztgPy+ft9tBFXoOlDNiOT9WUXZlA== -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: - name wrap-ansi-cjs +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -7133,6 +7148,15 @@ wrap-ansi@^6.2.0: string-width "^4.1.0" strip-ansi "^6.0.0" +wrap-ansi@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" + integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== + dependencies: + ansi-styles "^4.0.0" + string-width "^4.1.0" + strip-ansi "^6.0.0" + wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214" From 26b03629a14779156aeb638abdf19313c146591a Mon Sep 17 00:00:00 2001 From: Mickael Bourgois Date: Mon, 21 Jul 2025 16:58:58 +0200 Subject: [PATCH 09/10] CLDSRV-720: Add back file test null compat matrix - Skip Restore tests for file v1 cf: CLDSRV-721 --- .github/workflows/tests.yaml | 6 +++++- .../aws-node-sdk/test/object/mpuVersion.js | 8 ++++++-- .../aws-node-sdk/test/object/putVersion.js | 8 ++++++-- tests/functional/aws-node-sdk/test/utils/init.js | 14 ++++++++++++++ 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 36c702b832..29d157cbc4 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -395,7 +395,10 @@ jobs: strategy: matrix: include: - - job-name: file-ft-tests + - enable-null-compat: '' + job-name: file-ft-tests + - enable-null-compat: 'true' + job-name: file-ft-tests-null-compat name: ${{ matrix.job-name }} runs-on: ubuntu-24.04 needs: build @@ -405,6 +408,7 @@ jobs: CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }} MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }} MPU_TESTING: "yes" + ENABLE_NULL_VERSION_COMPAT_MODE: "${{ matrix.enable-null-compat }}" JOB_NAME: ${{ matrix.job-name }} steps: - name: Checkout diff --git a/tests/functional/aws-node-sdk/test/object/mpuVersion.js b/tests/functional/aws-node-sdk/test/object/mpuVersion.js index e175750a28..83849d9049 100644 --- a/tests/functional/aws-node-sdk/test/object/mpuVersion.js +++ b/tests/functional/aws-node-sdk/test/object/mpuVersion.js @@ -6,7 +6,7 @@ const BucketUtility = require('../../lib/utility/bucket-util'); const metadata = require('../../../../../lib/metadata/wrapper'); const { DummyRequestLogger } = require('../../../../unit/helpers'); const checkError = require('../../lib/utility/checkError'); -const { getMetadata, fakeMetadataArchive } = require('../utils/init'); +const { getMetadata, fakeMetadataArchive, isNullKeyMetadataV1 } = require('../utils/init'); const log = new DummyRequestLogger(); @@ -96,7 +96,11 @@ function checkObjMdAndUpdate(objMDBefore, objMDAfter, props) { }); } -describe('MPU with x-scal-s3-version-id header', () => { +// TODO: CLDSRV-721 RING 10 Support ObjectRestore (cold storage) with MD v1 +// The whole test suite is skipped as bad versionId breaks after each bucket cleanup +const describeSkipNullMdV1 = isNullKeyMetadataV1 ? describe.skip : describe; + +describeSkipNullMdV1('MPU with x-scal-s3-version-id header', () => { withV4(sigCfg => { let bucketUtil; let s3; diff --git a/tests/functional/aws-node-sdk/test/object/putVersion.js b/tests/functional/aws-node-sdk/test/object/putVersion.js index f6c74009c7..1c16f92ab2 100644 --- a/tests/functional/aws-node-sdk/test/object/putVersion.js +++ b/tests/functional/aws-node-sdk/test/object/putVersion.js @@ -6,7 +6,7 @@ const BucketUtility = require('../../lib/utility/bucket-util'); const metadata = require('../../../../../lib/metadata/wrapper'); const { DummyRequestLogger } = require('../../../../unit/helpers'); const checkError = require('../../lib/utility/checkError'); -const { getMetadata, fakeMetadataArchive } = require('../utils/init'); +const { getMetadata, fakeMetadataArchive, isNullKeyMetadataV1 } = require('../utils/init'); const log = new DummyRequestLogger(); @@ -47,7 +47,11 @@ function checkObjMdAndUpdate(objMDBefore, objMDAfter, props) { }); } -describe('PUT object with x-scal-s3-version-id header', () => { +// TODO: CLDSRV-721 RING 10 Support ObjectRestore (cold storage) with MD v1 +// The whole test suite is skipped as bad versionId breaks after each bucket cleanup +const describeSkipNullMdV1 = isNullKeyMetadataV1 ? describe.skip : describe; + +describeSkipNullMdV1('PUT object with x-scal-s3-version-id header', () => { withV4(sigCfg => { let bucketUtil; let s3; diff --git a/tests/functional/aws-node-sdk/test/utils/init.js b/tests/functional/aws-node-sdk/test/utils/init.js index e700c9d79e..7bcb86c448 100644 --- a/tests/functional/aws-node-sdk/test/utils/init.js +++ b/tests/functional/aws-node-sdk/test/utils/init.js @@ -7,6 +7,19 @@ const log = new DummyRequestLogger(); const nonVersionedObjId = versionIdUtils.getInfVid(config.replicationGroupId); +const isMetadataOrFile = ['file', 'scality'].includes(config.backends.metadata); +/** + * With null version compat mode the null key should look like + * 'object1putversion\u000099999999999999999999RG001 ' + * Without it for BucketFile backends it looks like + * 'object1putversion\x00' + * + * The later case does not support ObjectRestore and needs some tests to be skipped. + * + * TODO: CLDSRV-721 RING 10 Support ObjectRestore (cold storage) with MD v1 + */ +const isNullKeyMetadataV1 = isMetadataOrFile && !config.nullVersionCompatMode; + function decodeVersionId(versionId) { let decodedVersionId; if (versionId) { @@ -87,6 +100,7 @@ function fakeMetadataArchive(bucketName, objectName, versionId, archive, cb) { } module.exports = { + isNullKeyMetadataV1, initMetadata, getMetadata, fakeMetadataArchive, From eb292e2899890e40aa9aa3be6acb1c7244f5a520 Mon Sep 17 00:00:00 2001 From: Mickael Bourgois Date: Tue, 22 Jul 2025 10:56:00 +0200 Subject: [PATCH 10/10] CLDSRV-720: Lint replace tabs with spaces --- .../aws-node-sdk/test/utils/init.js | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/functional/aws-node-sdk/test/utils/init.js b/tests/functional/aws-node-sdk/test/utils/init.js index 7bcb86c448..dec227aab2 100644 --- a/tests/functional/aws-node-sdk/test/utils/init.js +++ b/tests/functional/aws-node-sdk/test/utils/init.js @@ -35,16 +35,16 @@ function decodeVersionId(versionId) { let metadataInit = false; function initMetadata(done) { - if (metadataInit === true) { - return done(); - } - return metadata.setup(err => { - if (err) { - return done(err); - } - metadataInit = true; - return done(); - }); + if (metadataInit === true) { + return done(); + } + return metadata.setup(err => { + if (err) { + return done(err); + } + metadataInit = true; + return done(); + }); } function getMetadata(bucketName, objectName, versionId, cb) { @@ -64,8 +64,8 @@ function getMetadata(bucketName, objectName, versionId, cb) { function fakeMetadataTransition(bucketName, objectName, versionId, cb) { return getMetadata(bucketName, objectName, versionId, (err, objMD) => { if (err) { - return cb(err); - } + return cb(err); + } /* eslint-disable no-param-reassign */ objMD['x-amz-scal-transition-in-progress'] = true; /* eslint-enable no-param-reassign */ @@ -87,8 +87,8 @@ function fakeMetadataTransition(bucketName, objectName, versionId, cb) { function fakeMetadataArchive(bucketName, objectName, versionId, archive, cb) { return getMetadata(bucketName, objectName, versionId, (err, objMD) => { if (err) { - return cb(err); - } + return cb(err); + } /* eslint-disable no-param-reassign */ objMD['x-amz-storage-class'] = 'location-dmf-v1'; objMD.dataStoreName = 'location-dmf-v1'; @@ -101,8 +101,8 @@ function fakeMetadataArchive(bucketName, objectName, versionId, archive, cb) { module.exports = { isNullKeyMetadataV1, - initMetadata, - getMetadata, - fakeMetadataArchive, + initMetadata, + getMetadata, + fakeMetadataArchive, fakeMetadataTransition, };