Merge remote-tracking branch 'origin/bugfix/CLDSRV-665' into w/9.0/bugfix/CLDSRV-665

Author: Kerkesni

.github/docker/docker-compose.yaml

@@ -60,6 +60,7 @@ services:
       - SCUBA_HEALTHCHECK_FREQUENCY
       - S3QUOTA
       - QUOTA_ENABLE_INFLIGHTS
+      - S3_VERSION_ID_ENCODING_TYPE
     env_file:
       - creds.env
     depends_on:

.github/workflows/tests.yaml

@@ -344,6 +344,7 @@ jobs:
       S3METADATA: mongodb
       S3KMS: file
       S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
+      S3_VERSION_ID_ENCODING_TYPE: base62
       DEFAULT_BUCKET_KEY_FORMAT: v1
       METADATA_MAX_CACHED_BUCKETS: 1
       MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}

lib/Config.js

@@ -910,6 +910,38 @@ class Config extends EventEmitter {
             this.replicationGroupId = 'RG001';
         }
 
+        const instanceId = process.env.CLOUDSERVER_INSTANCE_ID || config.instanceId;
+        if (instanceId) {
+            assert(typeof instanceId === 'string',
+                'bad config: instanceId must be a string');
+            // versionID generation code will truncate instanceId to 6 characters
+            // so we enforce this limit here to make the behavior predictable
+            assert(instanceId.length <= 6,
+                'bad config: instanceId must be at most 6 characters long');
+            this.instanceId = instanceId;
+        } else if (process.env.HOSTNAME) {
+            // If running in Kubernetes, the pod name is set in the HOSTNAME env var
+            // and is of the form: cloudserver-7869f99955-nr984
+            // where:
+            // - cloudserver is the name of the pod
+            // - 7869f99955 is a long hash that guarantees uniqueness within the namespace
+            // - nr984 is a short hash that is unique only within the current ReplicaSet/Deployment
+            // We use the last part of the pod name as the instanceId, prefixed with 'i' for internal
+            // or 'e' for external cloudserver.
+            // This allows having a unique identifier that will be used when generating versionIds
+            const instanceType = process.env.HOSTNAME.includes('internal') ? 'i' : 'e';
+            const parts = process.env.HOSTNAME.split('-');
+            if (parts.length >= 2) {
+                const shortId = parts[parts.length - 1];
+                this.instanceId = instanceType + shortId;
+            } else {
+                // fallback: generate a random string if format is unexpected
+                this.instanceId = uuidv4().replace(/-/g, '').slice(0, 6);
+            }
+        } else {
+            this.instanceId = uuidv4().replace(/-/g, '').slice(0, 6);
+        }
+
         this.replicationEndpoints = [];
         if (config.replicationEndpoints) {
             const { replicationEndpoints } = config;

lib/metadata/wrapper.js

@@ -30,6 +30,7 @@ if (clientName === 'mem') {
     params = {
         mongodb: config.mongodb,
         replicationGroupId: config.replicationGroupId,
+        instanceId: config.instanceId,
         config,
     };
 } else if (clientName === 'cdmi') {

tests/unit/Config.js

@@ -1,6 +1,7 @@
 const assert = require('assert');
 const fs = require('fs');
 const sinon = require('sinon');
+const path = require('path');
 const {
     azureGetStorageAccountName,
     azureGetLocationCredentials,
@@ -810,4 +811,82 @@ describe('Config', () => {
             assert.deepEqual(config.internalListenOn, [{ ip: '127.0.0.1', port: 9000 }]);
         });
     });
+
+    describe('instanceId', () => {
+        let defaultConfig;
+
+        before(() => {
+            setEnv('S3_CONFIG_FILE', 'tests/unit/testConfigs/allOptsConfig/config.json');
+            // Import config
+            const defaultConfigPath = path.join(__dirname, './testConfigs/allOptsConfig/config.json');
+            defaultConfig = require(defaultConfigPath);
+        });
+
+        afterEach(() => {
+            sinon.restore();
+        });
+
+        it('should use the instance id set in the config', () => {
+            // Stub fs.readFileSync to return a specific config.json content that includes instanceId
+            const originalReadFileSync = fs.readFileSync;
+            const readFileSyncStub = sinon.stub(fs, 'readFileSync');
+            // Mock only config.json file
+            readFileSyncStub
+                .withArgs(sinon.match(/\/config\.json$/))
+                .returns(JSON.stringify({ ...defaultConfig, instanceId: 'test' }));
+            // For all other files, use the original readFileSync
+            readFileSyncStub
+                .callsFake((filePath, ...args) => originalReadFileSync(filePath, ...args));
+            // Create a new ConfigObject instance
+            const config = new ConfigObject();
+            assert.strictEqual(config.instanceId, 'test');
+        });
+
+        it('should assert if instanceId is not a string', () => {
+            // Stub fs.readFileSync to return a specific config.json content that includes instanceId
+            const originalReadFileSync = fs.readFileSync;
+            const readFileSyncStub = sinon.stub(fs, 'readFileSync');
+            // Mock only config.json file
+            readFileSyncStub
+                .withArgs(sinon.match(/\/config\.json$/))
+                .returns(JSON.stringify({ ...defaultConfig, instanceId: 1234 }));
+            // For all other files, use the original readFileSync
+            readFileSyncStub
+                .callsFake((filePath, ...args) => originalReadFileSync(filePath, ...args));
+            // Create a new ConfigObject instance
+            assert.throws(() => new ConfigObject());
+        });
+
+        it('should use the instance id set in the env var', () => {
+            setEnv('CLOUDSERVER_INSTANCE_ID', '123456');
+            const config = new ConfigObject();
+            assert.strictEqual(config.instanceId, '123456');
+        });
+
+        it('should assert if instanceId is longer than 6 characters', () => {
+            setEnv('CLOUDSERVER_INSTANCE_ID', '1234567');
+            assert.throws(() => new ConfigObject());
+        });
+
+        it('should generate a new instanceId for external cloudserver', () => {
+            setEnv('HOSTNAME', 'connector-cloudserver-69958b4697-bs5pn');
+            const config = new ConfigObject();
+            assert.strictEqual(config.instanceId, 'ebs5pn');
+        });
+
+        it('should generate a new instanceId for internal cloudserver', () => {
+            setEnv('HOSTNAME', 'internal-cloudserver-54dc76b796-48m2x');
+            const config = new ConfigObject();
+            assert.strictEqual(config.instanceId, 'i48m2x');
+        });
+        it('should generate a random instanceId if HOSTNAME has an unexpected format', () => {
+            setEnv('HOSTNAME', 'cldsrv1');
+            const config = new ConfigObject();
+            assert.strictEqual(config.instanceId.length, 6);
+        });
+        it('should generate a random instanceId if HOSTNAME is not set', () => {
+            const config = new ConfigObject();
+            assert.strictEqual(config.instanceId.length, 6);
+        });
+    });
 });