Skip to content

CLDSRV-720: Fix mocha run functional tests #5875

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bert-e merged 10 commits into from
Jul 23, 2025
Merged

CLDSRV-720: Fix mocha run functional tests #5875

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
29d2d54
CLDSRV-720: Fix mocha config to run ft tests
BourgoisMickael Jul 18, 2025
a20d248
CLDSRV-720: Fix import path in tests
BourgoisMickael Jul 18, 2025
ccbb774
CLDSRV-720: Fix replace bluebird with Promise
BourgoisMickael Jul 18, 2025
479edda
CLDSRV-720: Fix socket reused when counting bytes
BourgoisMickael Jul 18, 2025
c447d70
CLDSRV-720: Fix test HEAD and partNumber (of MPU)
BourgoisMickael Jul 18, 2025
26e0396
CLDSRV-720: Try to clean all buckets before list
BourgoisMickael Jul 18, 2025
f4346e0
CLDSRV-720: Fix test hook metadata mongoclient
BourgoisMickael Jul 18, 2025
4ed1bc2
CLDSRV-720: Bump arsenal to fix GCP createMPU
BourgoisMickael Jul 20, 2025
26b0362
CLDSRV-720: Add back file test null compat matrix
BourgoisMickael Jul 21, 2025
eb292e2
CLDSRV-720: Lint replace tabs with spaces
BourgoisMickael Jul 22, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion .github/workflows/tests.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -395,7 +395,10 @@ jobs:
strategy:
matrix:
include:
- job-name: file-ft-tests
- enable-null-compat: ''
job-name: file-ft-tests
- enable-null-compat: 'true'
job-name: file-ft-tests-null-compat
name: ${{ matrix.job-name }}
runs-on: ubuntu-24.04
needs: build
Expand All @@ -405,6 +408,7 @@ jobs:
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
MPU_TESTING: "yes"
ENABLE_NULL_VERSION_COMPAT_MODE: "${{ matrix.enable-null-compat }}"
JOB_NAME: ${{ matrix.job-name }}
steps:
- name: Checkout
Expand Down
6 changes: 5 additions & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
"dependencies": {
"@azure/storage-blob": "^12.25.0",
"@hapi/joi": "^17.1.1",
"arsenal": "git+https://github.com/scality/Arsenal#8.2.26",
"arsenal": "git+https://github.com/scality/Arsenal#8.2.27",
"async": "2.6.4",
"aws-sdk": "^2.1692.0",
"bucketclient": "scality/bucketclient#8.2.4",
Expand Down Expand Up @@ -76,6 +76,10 @@
"jsonwebtoken": "^9.0.0",
"nan": "v2.22.0"
},
"mocha": {
"recursive": true,
"timeout": 40000
},
"scripts": {
"cloudserver": "S3METADATA=mongodb npm-run-all --parallel start_dataserver start_s3server",
"ft_awssdk": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/ --exit",
Expand Down
204 changes: 80 additions & 124 deletions tests/functional/aws-node-sdk/test/bucket/get.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -337,7 +337,7 @@ describe('GET Bucket - AWS.S3.listObjects', () => {
});

afterEach(done => {
bucketUtil.empty(bucketName).catch(done).done(() => done());
bucketUtil.empty(bucketName).then(() => done()).catch(done);
});
Comment thread
This conversation was marked as resolved.

tests.forEach(test => {
Expand All @@ -357,181 +357,137 @@ describe('GET Bucket - AWS.S3.listObjects', () => {
});

tests.forEach(test => {
it(`v2 should ${test.name}`, done => {
it(`v2 should ${test.name}`, async () => {
const s3 = bucketUtil.s3;
const Bucket = bucketName;

Promise
.mapSeries(test.objectPutParams(Bucket),
param => s3.putObject(param).promise())
.then(() =>
s3.listObjectsV2(test.listObjectParams(Bucket))
.promise())
.then(data => {
const isValidResponse =
tv4.validate(data, bucketSchemaV2);
if (!isValidResponse) {
throw new Error(tv4.error);
}
return data;
}).then(data => {
test.assertions(data, Bucket);
done();
})
.catch(done);
for (const param of test.objectPutParams(Bucket)) {
await s3.putObject(param).promise();
}
const data = await s3.listObjectsV2(test.listObjectParams(Bucket)).promise();
const isValidResponse = tv4.validate(data, bucketSchemaV2);
if (!isValidResponse) {
throw new Error(tv4.error);
}
test.assertions(data, Bucket);
});
});

['&amp', '"quot', '\'apos', '<lt', '>gt'].forEach(k => {
it(`should list objects with key ${k} as Prefix`, done => {
it(`should list objects with key ${k} as Prefix`, async () => {
const s3 = bucketUtil.s3;
const Bucket = bucketName;
const objects = [{ Bucket, Key: k }];

Promise
.mapSeries(objects, param => s3.putObject(param).promise())
.then(() => s3.listObjects({ Bucket, Prefix: k }).promise())
.then(data => {
const isValidResponse = tv4.validate(data,
bucketSchema);
if (!isValidResponse) {
throw new Error(tv4.error);
}
return data;
}).then(data => {
assert.deepStrictEqual(data.Prefix, k);
done();
})
.catch(done);
for (const param of objects) {
await s3.putObject(param).promise();
}
const data = await s3.listObjects({ Bucket, Prefix: k }).promise();
const isValidResponse = tv4.validate(data, bucketSchema);
if (!isValidResponse) {
throw new Error(tv4.error);
}
assert.deepStrictEqual(data.Prefix, k);
});
});

['&amp', '"quot', '\'apos', '<lt', '>gt'].forEach(k => {
it(`should list objects with key ${k} as Marker`, done => {
it(`should list objects with key ${k} as Marker`, async () => {
const s3 = bucketUtil.s3;
const Bucket = bucketName;
const objects = [{ Bucket, Key: k }];

Promise
.mapSeries(objects, param => s3.putObject(param).promise())
.then(() => s3.listObjects({ Bucket, Marker: k }).promise())
.then(data => {
const isValidResponse = tv4.validate(data,
bucketSchema);
if (!isValidResponse) {
throw new Error(tv4.error);
}
return data;
}).then(data => {
assert.deepStrictEqual(data.Marker, k);
done();
})
.catch(done);
for (const param of objects) {
await s3.putObject(param).promise();
}
const data = await s3.listObjects({ Bucket, Marker: k }).promise();
const isValidResponse = tv4.validate(data, bucketSchema);
if (!isValidResponse) {
throw new Error(tv4.error);
}
assert.deepStrictEqual(data.Marker, k);
});
});

['&amp', '"quot', '\'apos', '<lt', '>gt'].forEach(k => {
it(`should list objects with key ${k} as NextMarker`, done => {
it(`should list objects with key ${k} as NextMarker`, async () => {
const s3 = bucketUtil.s3;
const Bucket = bucketName;
const objects = [{ Bucket, Key: k }, { Bucket, Key: 'zzz' }];

Promise
.mapSeries(objects, param => s3.putObject(param).promise())
.then(() => s3.listObjects({ Bucket, MaxKeys: 1,
Delimiter: 'foo' }).promise())
.then(data => {
const isValidResponse = tv4.validate(data,
bucketSchema);
if (!isValidResponse) {
throw new Error(tv4.error);
}
return data;
}).then(data => {
assert.strictEqual(data.NextMarker, k);
done();
})
.catch(done);
for (const param of objects) {
await s3.putObject(param).promise();
}
const data = await s3.listObjects({ Bucket, MaxKeys: 1,
Delimiter: 'foo' }).promise();
const isValidResponse = tv4.validate(data, bucketSchema);
if (!isValidResponse) {
throw new Error(tv4.error);
}
assert.strictEqual(data.NextMarker, k);
});
});

['&amp', '"quot', '\'apos', '<lt', '>gt'].forEach(k => {
it(`should list objects with key ${k} as StartAfter`, done => {
it(`should list objects with key ${k} as StartAfter`, async () => {
const s3 = bucketUtil.s3;
const Bucket = bucketName;
const objects = [{ Bucket, Key: k }];

Promise
.mapSeries(objects, param => s3.putObject(param).promise())
.then(() => s3.listObjectsV2(
{ Bucket, StartAfter: k }).promise())
.then(data => {
const isValidResponse = tv4.validate(data,
bucketSchemaV2);
if (!isValidResponse) {
throw new Error(tv4.error);
}
return data;
}).then(data => {
assert.deepStrictEqual(data.StartAfter, k);
done();
})
.catch(done);
for (const param of objects) {
await s3.putObject(param).promise();
}
const data = await s3.listObjectsV2(
{ Bucket, StartAfter: k }).promise();
const isValidResponse = tv4.validate(data, bucketSchemaV2);
if (!isValidResponse) {
throw new Error(tv4.error);
}
assert.deepStrictEqual(data.StartAfter, k);
});
});

['&amp', '"quot', '\'apos', '<lt', '>gt'].forEach(k => {
it(`should list objects with key ${k} as ContinuationToken`,
done => {
async () => {
const s3 = bucketUtil.s3;
const Bucket = bucketName;
const objects = [{ Bucket, Key: k }];

Promise
.mapSeries(objects, param => s3.putObject(param).promise())
.then(() => s3.listObjectsV2({
Bucket,
ContinuationToken: generateToken(k),
}).promise())
.then(data => {
const isValidResponse = tv4.validate(data,
bucketSchemaV2);
if (!isValidResponse) {
throw new Error(tv4.error);
}
return data;
}).then(data => {
assert.deepStrictEqual(
decryptToken(data.ContinuationToken), k);
done();
})
.catch(done);
for (const param of objects) {
await s3.putObject(param).promise();
}
const data = await s3.listObjectsV2({
Bucket,
ContinuationToken: generateToken(k),
}).promise();
const isValidResponse = tv4.validate(data, bucketSchemaV2);
if (!isValidResponse) {
throw new Error(tv4.error);
}
assert.deepStrictEqual(
decryptToken(data.ContinuationToken), k);
});
});

['&amp', '"quot', '\'apos', '<lt', '>gt'].forEach(k => {
it(`should list objects with key ${k} as NextContinuationToken`,
done => {
async () => {
const s3 = bucketUtil.s3;
const Bucket = bucketName;
const objects = [{ Bucket, Key: k }, { Bucket, Key: 'zzz' }];
Promise
.mapSeries(objects, param => s3.putObject(param).promise())
.then(() => s3.listObjectsV2({ Bucket, MaxKeys: 1,
Delimiter: 'foo' }).promise())
.then(data => {
const isValidResponse = tv4.validate(data,
bucketSchemaV2);
if (!isValidResponse) {
throw new Error(tv4.error);
}
return data;
}).then(data => {
assert.strictEqual(
decryptToken(data.NextContinuationToken), k);
done();
})
.catch(done);

for (const param of objects) {
await s3.putObject(param).promise();
}
const data = await s3.listObjectsV2({ Bucket, MaxKeys: 1,
Delimiter: 'foo' }).promise();
const isValidResponse = tv4.validate(data, bucketSchemaV2);
if (!isValidResponse) {
throw new Error(tv4.error);
}
assert.strictEqual(
decryptToken(data.NextContinuationToken), k);
});
});
});
Expand Down
2 changes: 1 addition & 1 deletion tests/functional/aws-node-sdk/test/bucket/getBucketEncryption.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ describe('aws-sdk test get bucket encryption', () => {
before(done => {
const config = getConfig('default', { signatureVersion: 'v4' });
s3 = new S3(config);
return metadata.setup(done);
metadata.setup(done);
});

beforeEach(done => s3.createBucket({ Bucket: bucketName }, done));
Expand Down
6 changes: 3 additions & 3 deletions tests/functional/aws-node-sdk/test/bucket/put.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -257,9 +257,9 @@ describe('PUT Bucket - AWS.S3.createBucket', () => {
location => {
describeSkipAWS(`bucket creation with location: ${location}`,
() => {
after(done =>
bucketUtil.deleteOne(bucketName)
.then(() => done()).catch(() => done()));
after(done => {
bucketUtil.deleteOne(bucketName).finally(done);
});
it(`should create bucket with location: ${location}`, done => {
bucketUtil.s3.createBucket(
{
Expand Down
3 changes: 0 additions & 3 deletions tests/functional/aws-node-sdk/test/mocha.opts
View file
Open in desktop

This file was deleted.

2 changes: 2 additions & 0 deletions tests/functional/aws-node-sdk/test/object/100-continue.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ class ContinueRequestHandler {

getRequestOptions() {
return {
// Prevent socket reuse as a test checks for socket.bytesWritten
agent: new http.Agent({ keepAlive: false }),
Comment thread
BourgoisMickael marked this conversation as resolved.
path: this.path,
hostname,
port,
Expand Down
18 changes: 13 additions & 5 deletions tests/functional/aws-node-sdk/test/object/getPartSize.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ const object = 'mpu-test-object';
const emptyObject = 'empty-object';
const nonMpuObject = 'simple-object';

/** 5MiB */
const bodySize = 1024 * 1024 * 5;
const bodyContent = 'a';
const howManyParts = 3;
Expand All @@ -18,6 +19,13 @@ const invalidPartNumbers = [-1, 0, maximumAllowedPartCount + 1];

let ETags = [];

// Because HEAD has no body, the SDK (v2) returns a generic code such as:
// 400 BadRequest
// 403 Forbidden
// 404 NotFound
// ...
// It will fall back to HTTP statusCode
// Example: 416 InvalidRange will be 416 416
function checkError(err, statusCode, code) {
assert.strictEqual(err.statusCode, statusCode);
assert.strictEqual(err.code, code);
Expand Down Expand Up @@ -174,7 +182,7 @@ describe('Part size tests with object head', () => {

it('should return an error when requesting part 2 of empty object', done => {
headObject({ Key: emptyObject, PartNumber: 2 }, (err, data) => {
checkError(err, 416, 'InvalidRange');
checkError(err, 416, 416);
Comment thread
BourgoisMickael marked this conversation as resolved.
assert.strictEqual(data, null);
done();
});
Expand All @@ -183,15 +191,15 @@ describe('Part size tests with object head', () => {
it('should return content-length requesting part 1 of non-MPU object', done => {
headObject({ Key: nonMpuObject, PartNumber: 1 }, (err, data) => {
checkNoError(err);
assert.strictEqual(data.ContentLength, 0);
assert.strictEqual(data.ContentLength, bodySize);
Comment thread
welansari marked this conversation as resolved.
done();
});
});

it('should return an error when requesting part 2 of non-MPU object', done => {
headObject({ Key: nonMpuObject, PartNumber: 1 }, (err, data) => {
checkError(err, 416, 'InvalidRange');
assert.strictEqual(data.ContentLength, bodySize);
headObject({ Key: nonMpuObject, PartNumber: 2 }, (err, data) => {
checkError(err, 416, 416);
Comment thread
BourgoisMickael marked this conversation as resolved.
assert.strictEqual(data, null);
done();
});
});
Expand Down
Loading
Loading