Skip to content

Commit 1821c3a

Browse files
committed
Merge branches 'w/9.1/bugfix/CLDSRV-665' and 'q/w/5826/9.0/bugfix/CLDSRV-665' into tmp/octopus/q/9.1
3 parents cd4b15e + 36d29ab + e47d49e commit 1821c3a

7 files changed

Lines changed: 125 additions & 10 deletions

File tree

.github/docker/docker-compose.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ services:
6060
- SCUBA_HEALTHCHECK_FREQUENCY
6161
- S3QUOTA
6262
- QUOTA_ENABLE_INFLIGHTS
63+
- S3_VERSION_ID_ENCODING_TYPE
6364
env_file:
6465
- creds.env
6566
depends_on:

.github/workflows/tests.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -344,6 +344,7 @@ jobs:
344344
S3METADATA: mongodb
345345
S3KMS: file
346346
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
347+
S3_VERSION_ID_ENCODING_TYPE: base62
347348
DEFAULT_BUCKET_KEY_FORMAT: v1
348349
METADATA_MAX_CACHED_BUCKETS: 1
349350
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}

lib/Config.js

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -910,6 +910,38 @@ class Config extends EventEmitter {
910910
this.replicationGroupId = 'RG001';
911911
}
912912

913+
const instanceId = process.env.CLOUDSERVER_INSTANCE_ID || config.instanceId;
914+
if (instanceId) {
915+
assert(typeof instanceId === 'string',
916+
'bad config: instanceId must be a string');
917+
// versionID generation code will truncate instanceId to 6 characters
918+
// so we enforce this limit here to make the behavior predictable
919+
assert(instanceId.length <= 6,
920+
'bad config: instanceId must be at most 6 characters long');
921+
this.instanceId = instanceId;
922+
} else if (process.env.HOSTNAME) {
923+
// If running in Kubernetes, the pod name is set in the HOSTNAME env var
924+
// and is of the form: cloudserver-7869f99955-nr984
925+
// where:
926+
// - cloudserver is the name of the pod
927+
// - 7869f99955 is a long hash that guarantees uniqueness within the namespace
928+
// - nr984 is a short hash that is unique only within the current ReplicaSet/Deployment
929+
// We use the last part of the pod name as the instanceId, prefixed with 'i' for internal
930+
// or 'e' for external cloudserver.
931+
// This allows having a unique identifier that will be used when generating versionIds
932+
const instanceType = process.env.HOSTNAME.includes('internal') ? 'i' : 'e';
933+
const parts = process.env.HOSTNAME.split('-');
934+
if (parts.length >= 2) {
935+
const shortId = parts[parts.length - 1];
936+
this.instanceId = instanceType + shortId;
937+
} else {
938+
// fallback: generate a random string if format is unexpected
939+
this.instanceId = uuidv4().replace(/-/g, '').slice(0, 6);
940+
}
941+
} else {
942+
this.instanceId = uuidv4().replace(/-/g, '').slice(0, 6);
943+
}
944+
913945
this.replicationEndpoints = [];
914946
if (config.replicationEndpoints) {
915947
const { replicationEndpoints } = config;

lib/metadata/wrapper.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ if (clientName === 'mem') {
3030
params = {
3131
mongodb: config.mongodb,
3232
replicationGroupId: config.replicationGroupId,
33+
instanceId: config.instanceId,
3334
config,
3435
};
3536
} else if (clientName === 'cdmi') {

package.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@zenko/cloudserver",
3-
"version": "9.0.20",
3+
"version": "9.0.21",
44
"description": "Zenko CloudServer, an open-source Node.js implementation of a server handling the Amazon S3 protocol",
55
"main": "index.js",
66
"engines": {
@@ -21,7 +21,7 @@
2121
"dependencies": {
2222
"@azure/storage-blob": "^12.25.0",
2323
"@hapi/joi": "^17.1.1",
24-
"arsenal": "git+https://github.com/scality/Arsenal#8.2.25",
24+
"arsenal": "git+https://github.com/scality/Arsenal#8.2.26",
2525
"async": "2.6.4",
2626
"aws-sdk": "^2.1692.0",
2727
"bucketclient": "scality/bucketclient#8.2.4",

tests/unit/Config.js

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
const assert = require('assert');
22
const fs = require('fs');
33
const sinon = require('sinon');
4+
const path = require('path');
45
const {
56
azureGetStorageAccountName,
67
azureGetLocationCredentials,
@@ -814,4 +815,82 @@ describe('Config', () => {
814815
assert.deepEqual(config.internalListenOn, [{ ip: '127.0.0.1', port: 9000 }]);
815816
});
816817
});
818+
819+
describe('instanceId', () => {
820+
let defaultConfig;
821+
822+
before(() => {
823+
setEnv('S3_CONFIG_FILE', 'tests/unit/testConfigs/allOptsConfig/config.json');
824+
// Import config
825+
const defaultConfigPath = path.join(__dirname, './testConfigs/allOptsConfig/config.json');
826+
defaultConfig = require(defaultConfigPath);
827+
});
828+
829+
afterEach(() => {
830+
sinon.restore();
831+
});
832+
833+
it('should use the instance id set in the config', () => {
834+
// Stub fs.readFileSync to return a specific config.json content that includes instanceId
835+
const originalReadFileSync = fs.readFileSync;
836+
const readFileSyncStub = sinon.stub(fs, 'readFileSync');
837+
// Mock only config.json file
838+
readFileSyncStub
839+
.withArgs(sinon.match(/\/config\.json$/))
840+
.returns(JSON.stringify({ ...defaultConfig, instanceId: 'test' }));
841+
// For all other files, use the original readFileSync
842+
readFileSyncStub
843+
.callsFake((filePath, ...args) => originalReadFileSync(filePath, ...args));
844+
// Create a new ConfigObject instance
845+
const config = new ConfigObject();
846+
assert.strictEqual(config.instanceId, 'test');
847+
});
848+
849+
it('should assert if instanceId is not a string', () => {
850+
// Stub fs.readFileSync to return a specific config.json content that includes instanceId
851+
const originalReadFileSync = fs.readFileSync;
852+
const readFileSyncStub = sinon.stub(fs, 'readFileSync');
853+
// Mock only config.json file
854+
readFileSyncStub
855+
.withArgs(sinon.match(/\/config\.json$/))
856+
.returns(JSON.stringify({ ...defaultConfig, instanceId: 1234 }));
857+
// For all other files, use the original readFileSync
858+
readFileSyncStub
859+
.callsFake((filePath, ...args) => originalReadFileSync(filePath, ...args));
860+
// Create a new ConfigObject instance
861+
assert.throws(() => new ConfigObject());
862+
});
863+
864+
it('should use the instance id set in the env var', () => {
865+
setEnv('CLOUDSERVER_INSTANCE_ID', '123456');
866+
const config = new ConfigObject();
867+
assert.strictEqual(config.instanceId, '123456');
868+
});
869+
870+
it('should assert if instanceId is longer than 6 characters', () => {
871+
setEnv('CLOUDSERVER_INSTANCE_ID', '1234567');
872+
assert.throws(() => new ConfigObject());
873+
});
874+
875+
it('should generate a new instanceId for external cloudserver', () => {
876+
setEnv('HOSTNAME', 'connector-cloudserver-69958b4697-bs5pn');
877+
const config = new ConfigObject();
878+
assert.strictEqual(config.instanceId, 'ebs5pn');
879+
});
880+
881+
it('should generate a new instanceId for internal cloudserver', () => {
882+
setEnv('HOSTNAME', 'internal-cloudserver-54dc76b796-48m2x');
883+
const config = new ConfigObject();
884+
assert.strictEqual(config.instanceId, 'i48m2x');
885+
});
886+
it('should generate a random instanceId if HOSTNAME has an unexpected format', () => {
887+
setEnv('HOSTNAME', 'cldsrv1');
888+
const config = new ConfigObject();
889+
assert.strictEqual(config.instanceId.length, 6);
890+
});
891+
it('should generate a random instanceId if HOSTNAME is not set', () => {
892+
const config = new ConfigObject();
893+
assert.strictEqual(config.instanceId.length, 6);
894+
});
895+
});
817896
});

yarn.lock

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -173,10 +173,10 @@
173173
fast-xml-parser "^5.0.7"
174174
tslib "^2.8.1"
175175

176-
"@azure/identity@^4.10.1":
177-
version "4.10.1"
178-
resolved "https://registry.yarnpkg.com/@azure/identity/-/identity-4.10.1.tgz#6ce1e7b367d2eb6680d0efa5817382aafcd86f05"
179-
integrity sha512-YM/z6RxRtFlXUH2egAYF/FDPes+MUE6ZoknjEdaq7ebJMMNUzn9zCJ3bd2ZZZlkP0r1xKa88kolhFH/FGV7JnA==
176+
"@azure/identity@^4.10.2":
177+
version "4.10.2"
178+
resolved "https://registry.yarnpkg.com/@azure/identity/-/identity-4.10.2.tgz#6609ce398824ff0bb53f1ad1043a9f1cc93e56b8"
179+
integrity sha512-Uth4vz0j+fkXCkbvutChUj03PDCokjbC6Wk9JT8hHEUtpy/EurNKAseb3+gO6Zi9VYBvwt61pgbzn1ovk942Qg==
180180
dependencies:
181181
"@azure/abort-controller" "^2.0.0"
182182
"@azure/core-auth" "^1.9.0"
@@ -1296,11 +1296,11 @@ arraybuffer.prototype.slice@^1.0.4:
12961296
get-intrinsic "^1.2.6"
12971297
is-array-buffer "^3.0.4"
12981298

1299-
"arsenal@git+https://github.com/scality/Arsenal#8.2.25":
1300-
version "8.2.25"
1301-
resolved "git+https://github.com/scality/Arsenal#43736fa785ef9207d7c40cab81c038eb58837f06"
1299+
"arsenal@git+https://github.com/scality/Arsenal#8.2.26":
1300+
version "8.2.26"
1301+
resolved "git+https://github.com/scality/Arsenal#80bb045e77ed9a5205af105820dc23b8351d3258"
13021302
dependencies:
1303-
"@azure/identity" "^4.10.1"
1303+
"@azure/identity" "^4.10.2"
13041304
"@azure/storage-blob" "^12.27.0"
13051305
"@js-sdsl/ordered-set" "^4.4.2"
13061306
"@scality/hdclient" "^1.3.1"
@@ -7115,6 +7115,7 @@ workerpool@^6.5.1:
71157115
integrity sha512-Fs4dNYcsdpYSAfVxhnl1L5zTksjvOJxtC5hzMNl+1t9B8hTJTdKDyZ5ju7ztgPy+ft9tBFXoOlDNiOT9WUXZlA==
71167116

71177117
"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0:
7118+
name wrap-ansi-cjs
71187119
version "7.0.0"
71197120
resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
71207121
integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==

0 commit comments

Comments
 (0)