CLDSRV-740: Add healthcheck internal route s3 port

.github/docker/config.s3c.json

@@ -22,7 +22,8 @@
     "clusters": 5,
     "kmsHideScalityArn": true,
     "healthChecks": {
-        "allowFrom": ["127.0.0.1", "::1"]
+        "allowFrom": ["127.0.0.1", "::1"],
+        "enableInternalRoute": true
     },
     "localCache": {
         "host": "localhost",

.github/workflows/tests.yaml

@@ -532,48 +532,38 @@ jobs:
     - name: Setup vault credentials like S3C Integration
       run: ./setup-s3c.sh
       working-directory: .github/docker
+    - name: Set config files env variables
+      run: |-
+        echo "S3_CONFIG_FILE=${{ github.workspace }}/.github/docker/config.s3c.json" >> $GITHUB_ENV
+        echo "S3_LOCATION_FILE=${{ github.workspace }}/tests/locationConfig/locationConfigS3C.json" >> $GITHUB_ENV
     - name: Run cloudserver-object tests
-      env:
-        S3_CONFIG_FILE: ${{ github.workspace }}/.github/docker/config.s3c.json
-        S3_LOCATION_FILE: ${{ github.workspace }}/tests/locationConfig/locationConfigS3C.json
       run: |-
         set -o pipefail;
         yarn run ft_awssdk_objects_misc | tee /tmp/artifacts/${{ matrix.job-name }}/ft_awssdk_objects_misc.log
     - name: Run cloudserver-version tests
-      env:
-        S3_CONFIG_FILE: ${{ github.workspace }}/.github/docker/config.s3c.json
-        S3_LOCATION_FILE: ${{ github.workspace }}/tests/locationConfig/locationConfigS3C.json
       run: |-
         set -o pipefail;
         yarn run ft_awssdk_versioning | tee /tmp/artifacts/${{ matrix.job-name }}/ft_awssdk_versioning.log
     - name: Run cloudserver-bucket tests
-      env:
-        S3_CONFIG_FILE: ${{ github.workspace }}/.github/docker/config.s3c.json
-        S3_LOCATION_FILE: ${{ github.workspace }}/tests/locationConfig/locationConfigS3C.json
       run: |-
         set -o pipefail;
         yarn run ft_awssdk_buckets | tee /tmp/artifacts/${{ matrix.job-name }}/ft_awssdk_buckets.log
     - name: Run cloudserver-routes (metadata) tests
-      env:
-        S3_CONFIG_FILE: ${{ github.workspace }}/.github/docker/config.s3c.json
-        S3_LOCATION_FILE: ${{ github.workspace }}/tests/locationConfig/locationConfigS3C.json
       run: |-
         set -o pipefail;
         yarn run ft_node_routes | tee /tmp/artifacts/${{ matrix.job-name }}/ft_node_routes.log
     - name: Run backbeat route tests
-      env:
-        S3_CONFIG_FILE: ${{ github.workspace }}/.github/docker/config.s3c.json
-        S3_LOCATION_FILE: ${{ github.workspace }}/tests/locationConfig/locationConfigS3C.json
       run: |-
         set -o pipefail;
         yarn run ft_route_backbeat | tee /tmp/artifacts/${{ matrix.job-name }}/ft_route_backbeat.log
     - name: Run backbeat tests
-      env:
-        S3_CONFIG_FILE: ${{ github.workspace }}/.github/docker/config.s3c.json
-        S3_LOCATION_FILE: ${{ github.workspace }}/tests/locationConfig/locationConfigS3C.json
       run: |-
         set -o pipefail;
         yarn run ft_backbeat | tee /tmp/artifacts/${{ matrix.job-name }}/ft_backbeat.log
+    - name: Run healthchecks tests
+      run: |-
+        set -o pipefail;
+        yarn run ft_healthchecks | tee /tmp/artifacts/${{ matrix.job-name }}/ft_healthchecks.log
     - name: Teardown CI services
       run: docker compose down redis sproxyd metadata-standalone vault-sse-before-migration cloudserver-sse-before-migration
       working-directory: .github/docker

lib/Config.js

@@ -1548,6 +1548,11 @@ class Config extends EventEmitter {
             this.healthChecks.allowFrom = defaultHealthChecks.allowFrom
                 .concat(config.healthChecks.allowFrom);
         }
+        /**
+         * CLDSRV-740: S3C with nginx s3-frontend needs the healthcheck on the
+         * same port as s3 to allow nginx upstream failover based on healthcheck
+         */
+        this.healthChecks.enableInternalRoute = config.healthChecks?.enableInternalRoute || false;
 
         if (config.certFilePaths) {
             assert(typeof config.certFilePaths === 'object' &&

lib/server.js

@@ -209,8 +209,10 @@ class S3Server {
 
         switch (req.url) {
             case '/live':
+                healthcheckHandler(clientInfo.clientIP, req, res, log, statsClient, false);
+                break;
             case '/ready':
-                healthcheckHandler(clientInfo.clientIP, req, res, log, statsClient);
+                healthcheckHandler(clientInfo.clientIP, req, res, log, statsClient, true);
                 break;
 
             default:

lib/utilities/healthcheckHandler.js

@@ -109,9 +109,10 @@ function checkIP(clientIP) {
  * @param {object} res - http response object
  * @param {object} log - werelogs logger instance
  * @param {object} statsClient - StatsClient Instance
+ * @param {boolean} deep - whether the healthcheck is a deep healthcheck
  * @return {undefined}
  */
-function healthcheckHandler(clientIP, req, res, log, statsClient) {
+function healthcheckHandler(clientIP, req, res, log, statsClient, deep) {
     function healthcheckEndHandler(err, results) {
         writeResponse(res, err, log, results, error => {
             if (error) {
@@ -121,8 +122,6 @@ function healthcheckHandler(clientIP, req, res, log, statsClient) {
         });
     }
 
-    const deep = (req.url === '/ready');
-
     // Attach the apiMethod method to the request, so it can used by monitoring in the server
     // eslint-disable-next-line no-param-reassign
     req.apiMethod = deep ? 'deepHealthcheck' : 'healthcheck';

lib/utilities/internalHandlers.js

@@ -4,6 +4,8 @@ const routeWorkflowEngineOperator =
       require('../routes/routeWorkflowEngineOperator');
 const { reportHandler } = require('./reportHandler');
 const routeVeeam = require('../routes/routeVeeam').routeVeeam;
+const { healthcheckHandler } = require('./healthcheckHandler');
+const { config } = require('../Config');
 
 const internalHandlers = {
     backbeat: routeBackbeat,
@@ -13,6 +15,14 @@ const internalHandlers = {
     veeam: routeVeeam,
 };
 
+if (config.healthChecks.enableInternalRoute) {
+    // CLDSRV-740: S3C with the healthcheck on s3 port needs the internal route
+    internalHandlers.healthcheck = function s3InternalHealthcheckHandler(clientIP, req, res, log, statsClient) {
+        const deep = req.url === '/_/healthcheck/deep';
+        return healthcheckHandler(clientIP, req, res, log, statsClient, deep);
+    };
+}
+
 module.exports = {
     internalHandlers,
 };

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zenko/cloudserver",
-  "version": "9.0.25",
+  "version": "9.0.26",
   "description": "Zenko CloudServer, an open-source Node.js implementation of a server handling the Amazon S3 protocol",
   "main": "index.js",
   "engines": {

tests/functional/healthchecks/package.json

@@ -3,6 +3,7 @@
   "version": "0.1.0",
   "description": "Test-healthcheck",
   "main": "tests.js",
+  "private": true,
   "repository": "",
   "keywords": [
     "test"

tests/functional/healthchecks/test/checkRoutes.js

@@ -8,10 +8,11 @@ const async = require('async');
 const Redis = require('ioredis');
 
 const conf = require('../../config.json');
+const { config } = require('../../../../lib/Config');
 
 const redis = new Redis({
-    host: conf.localCache.host,
-    port: conf.localCache.port,
+    host: config.localCache.host,
+    port: config.localCache.port,
     // disable offline queue
     enableOfflineQueue: false,
 });
@@ -23,7 +24,7 @@ const transport = transportStr === 'http' ? http : https;
 const options = {
     host: conf.ipAddress,
     path: '/live',
-    port: 8002,
+    port: config.metricsPort || 8002,
 };
 
 function checkResult(expectedStatus, res) {
@@ -55,7 +56,7 @@ function makeDummyS3Request(cb) {
     const getOptions = deepCopy(options);
     getOptions.path = '/foo/bar';
     getOptions.method = 'GET';
-    getOptions.port = 8000;
+    getOptions.port = config.port || 8000;
     getOptions.agent = makeAgent();
     const req = transport.request(getOptions);
     req.end(() => cb());
@@ -106,6 +107,30 @@ describe('Healthcheck routes', () => {
     });
 });
 
+if (config.healthChecks.enableInternalRoute) {
+    describe('Healthcheck s3 port internal routes', () => {
+        it('should return 200 OK on GET request', done => {
+            const getOptions = deepCopy(options);
+            getOptions.method = 'GET';
+            getOptions.path = '/_/healthcheck';
+            getOptions.port = config.port || 8000;
+            getOptions.agent = makeAgent();
+            const req = transport.request(getOptions, makeChecker(200, done));
+            req.end();
+        });
+
+        it('should return 200 on deep GET request', done => {
+            const deepOptions = deepCopy(options);
+            deepOptions.method = 'GET';
+            deepOptions.path = '/_/healthcheck/deep';
+            deepOptions.port = config.port || 8000;
+            deepOptions.agent = makeAgent();
+            const req = transport.request(deepOptions, makeChecker(200, done));
+            req.end();
+        });
+    });
+}
+
 describe('Healthcheck stats', () => {
     const totalReqs = 5;
     beforeEach(done => {

tests/functional/raw-node/package.json

@@ -3,6 +3,7 @@
   "version": "0.1.0",
   "description": "Test-rawnode",
   "main": "tests.js",
+  "private": true,
   "repository": "",
   "keywords": [
     "test"

tests/functional/report/package.json

@@ -1,5 +1,6 @@
 {
     "name": "report-tests",
+    "private": true,
     "scripts": {
         "test": "mocha -t 40000 *.js"
     }