ADJUST: Reconnect after Server Restart (#2269)

* ADJUST: Reconnect after Server Restart

* ADJUST: Logging

* ADD: `--list-different` for prettier format check

* REMOVE: `--list-different`

* REMOVE: buildPrometheusJob

* FIX: Format

Author: NeoPlays

launcher/package.json

@@ -26,7 +26,7 @@
     "stereum": "concurrently \"npm:electron:serve\" \"npm:watch:css\"",
     "backend:watch": "nodemon --watch 'src/backend' --exec 'npm run electron:serve'",
     "format": "prettier .  --write",
-    "format:check": "prettier .  --check"
+    "format:check": "prettier . --check"
   },
   "dependencies": {
     "@headlessui/vue": "^1.7.23",

launcher/src/backend/NodeConnection.js

@@ -2111,10 +2111,10 @@ export class NodeConnection {
       this.taskManager.otherTasksHandler(ref, "trigger restart", true);
       await this.sshService.disconnect();
       await new Promise((resolve) => setTimeout(resolve, 10000)); // Wait for the disconnect to be fully done
-      const retry = { connected: false, counter: 0, maxTries: 300 };
+      const retry = { connected: false, counter: 0, maxTries: 60 };
       log.info("Connecting via SSH");
 
-      while (!retry.connected && retry.counter < retry.maxTries) {
+      while (!retry.connected && retry.counter <= retry.maxTries) {
         try {
           retry.counter++;
           log.info(`Trying to connect (${retry.counter})`);
@@ -2131,10 +2131,11 @@ export class NodeConnection {
             true,
             err + "\n\n" + (retry.maxTries - retry.counter) + " tries left."
           );
-          log.info(" Could not connect.\n" + (retry.maxTries - retry.counter) + " tries left.");
+          log.info("Could not connect. " + (retry.maxTries - retry.counter) + " tries left.");
+          await new Promise((resolve) => setTimeout(resolve, 5000)); // Wait 5 seconds before retrying
         }
       }
-      log.info("OUT OF WHILE LOOP");
+
       if (retry.connected) {
         await this.establish(this.taskManager);
         this.taskManager.otherTasksHandler(ref, "Connected", true);

launcher/src/backend/ethereum-services/BesuService.js

@@ -80,10 +80,6 @@ export class BesuService extends NodeService {
     return "stereum-" + this.id + ":9545";
   }
 
-  buildPrometheusJob() {
-    return `\n  - job_name: stereum-${this.id}\n    static_configs:\n      - targets: [${this.buildExecutionClientMetricsEndpoint()}]`;
-  }
-
   getDataDir() {
     return this.volumes.find((volume) => volume.servicePath === "/opt/app/data")?.destinationPath;
   }

launcher/src/backend/ethereum-services/ErigonService.js

@@ -87,12 +87,6 @@ export class ErigonService extends NodeService {
     return "stereum-" + this.id + ":6060";
   }
 
-  buildPrometheusJob() {
-    return `\n  - job_name: stereum-${
-      this.id
-    }\n    metrics_path: /debug/metrics/prometheus\n    static_configs:\n      - targets: [${this.buildExecutionClientMetricsEndpoint()}]`;
-  }
-
   getDataDir() {
     return this.volumes.find((volume) => volume.servicePath === "/opt/data/erigon")?.destinationPath;
   }

launcher/src/backend/ethereum-services/GethService.js

@@ -97,12 +97,6 @@ export class GethService extends NodeService {
     return "stereum-" + this.id + ":6060";
   }
 
-  buildPrometheusJob() {
-    return `\n  - job_name: stereum-${
-      this.id
-    }\n    metrics_path: /debug/metrics/prometheus\n    static_configs:\n      - targets: [${this.buildExecutionClientMetricsEndpoint()}]`;
-  }
-
   getDataDir() {
     return this.volumes.find((volume) => volume.servicePath === "/opt/data/geth")?.destinationPath;
   }

launcher/src/backend/ethereum-services/GrandineBeaconService.js

@@ -103,10 +103,6 @@ export class GrandineBeaconService extends NodeService {
     return "stereum-" + this.id + ":5054";
   }
 
-  buildPrometheusJob() {
-    return `\n  - job_name: stereum-${this.id}\n    static_configs:\n      - targets: [${this.buildConsensusClientMetricsEndpoint()}]`;
-  }
-
   getDataDir() {
     return this.volumes.find((volume) => volume.servicePath === "/opt/app/data")?.destinationPath;
   }

launcher/src/backend/ethereum-services/LighthouseBeaconService.js

@@ -107,10 +107,6 @@ export class LighthouseBeaconService extends NodeService {
     return "stereum-" + this.id + ":5054";
   }
 
-  buildPrometheusJob() {
-    return `\n  - job_name: stereum-${this.id}\n    static_configs:\n      - targets: [${this.buildConsensusClientMetricsEndpoint()}]`;
-  }
-
   getDataDir() {
     return this.volumes.find((volume) => volume.servicePath === "/opt/app/beacon")?.destinationPath;
   }

launcher/src/backend/ethereum-services/LodestarBeaconService.js

@@ -99,12 +99,6 @@ export class LodestarBeaconService extends NodeService {
     return "stereum-" + this.id + ":8008";
   }
 
-  buildPrometheusJob() {
-    return `\n  - job_name: stereum-${
-      this.id
-    }\n    metrics_path: /metrics\n    static_configs:\n      - targets: [${this.buildConsensusClientMetricsEndpoint()}]`;
-  }
-
   getDataDir() {
     return this.volumes.find((volume) => volume.servicePath === "/opt/app/beacon")?.destinationPath;
   }

launcher/src/backend/ethereum-services/NethermindService.js

@@ -80,10 +80,6 @@ export class NethermindService extends NodeService {
     return "stereum-" + this.id + ":6060";
   }
 
-  buildPrometheusJob() {
-    return `\n  - job_name: stereum-${this.id}\n    static_configs:\n      - targets: [${this.buildExecutionClientMetricsEndpoint()}]`;
-  }
-
   getDataDir() {
     return this.volumes.find((volume) => volume.servicePath === "/opt/app/data")?.destinationPath;
   }

launcher/src/backend/ethereum-services/NimbusBeaconService.js

@@ -94,10 +94,6 @@ export class NimbusBeaconService extends NodeService {
     return "stereum-" + this.id + ":8008";
   }
 
-  buildPrometheusJob() {
-    return `\n  - job_name: "nimbus"\n    metrics_path: /metrics\n    static_configs:\n      - targets: [${this.buildConsensusClientMetricsEndpoint()}]`;
-  }
-
   getDataDir() {
     return this.volumes.find((volume) => volume.servicePath === "/opt/app/beacon")?.destinationPath;
   }