ADJUST: Reconnect after Server Restart

launcher/package.json

@@ -26,7 +26,7 @@
     "stereum": "concurrently \"npm:electron:serve\" \"npm:watch:css\"",
     "backend:watch": "nodemon --watch 'src/backend' --exec 'npm run electron:serve'",
     "format": "prettier .  --write",
-    "format:check": "prettier .  --check"
+    "format:check": "prettier . --check"
   },
   "dependencies": {
     "@headlessui/vue": "^1.7.23",

launcher/src/backend/NodeConnection.js

@@ -2111,10 +2111,10 @@ export class NodeConnection {
       this.taskManager.otherTasksHandler(ref, "trigger restart", true);
       await this.sshService.disconnect();
       await new Promise((resolve) => setTimeout(resolve, 10000)); // Wait for the disconnect to be fully done
-      const retry = { connected: false, counter: 0, maxTries: 300 };
+      const retry = { connected: false, counter: 0, maxTries: 60 };
       log.info("Connecting via SSH");
 
-      while (!retry.connected && retry.counter < retry.maxTries) {
+      while (!retry.connected && retry.counter <= retry.maxTries) {
         try {
           retry.counter++;
           log.info(`Trying to connect (${retry.counter})`);
@@ -2131,10 +2131,11 @@ export class NodeConnection {
             true,
             err + "\n\n" + (retry.maxTries - retry.counter) + " tries left."
           );
-          log.info(" Could not connect.\n" + (retry.maxTries - retry.counter) + " tries left.");
+          log.info("Could not connect. " + (retry.maxTries - retry.counter) + " tries left.");
+          await new Promise((resolve) => setTimeout(resolve, 5000)); // Wait 5 seconds before retrying
         }
       }
-      log.info("OUT OF WHILE LOOP");
+
       if (retry.connected) {
         await this.establish(this.taskManager);
         this.taskManager.otherTasksHandler(ref, "Connected", true);

launcher/src/backend/ethereum-services/BesuService.js

@@ -80,10 +80,6 @@ export class BesuService extends NodeService {
     return "stereum-" + this.id + ":9545";
   }
 
-  buildPrometheusJob() {
-    return `\n  - job_name: stereum-${this.id}\n    static_configs:\n      - targets: [${this.buildExecutionClientMetricsEndpoint()}]`;
-  }
-
   getDataDir() {
     return this.volumes.find((volume) => volume.servicePath === "/opt/app/data")?.destinationPath;
   }

launcher/src/backend/ethereum-services/ErigonService.js

@@ -87,12 +87,6 @@ export class ErigonService extends NodeService {
     return "stereum-" + this.id + ":6060";
   }
 
-  buildPrometheusJob() {
-    return `\n  - job_name: stereum-${
-      this.id
-    }\n    metrics_path: /debug/metrics/prometheus\n    static_configs:\n      - targets: [${this.buildExecutionClientMetricsEndpoint()}]`;
-  }
-
   getDataDir() {
     return this.volumes.find((volume) => volume.servicePath === "/opt/data/erigon")?.destinationPath;
   }

launcher/src/backend/ethereum-services/GethService.js

@@ -97,12 +97,6 @@ export class GethService extends NodeService {
     return "stereum-" + this.id + ":6060";
   }
 
-  buildPrometheusJob() {
-    return `\n  - job_name: stereum-${
-      this.id
-    }\n    metrics_path: /debug/metrics/prometheus\n    static_configs:\n      - targets: [${this.buildExecutionClientMetricsEndpoint()}]`;
-  }
-
   getDataDir() {
     return this.volumes.find((volume) => volume.servicePath === "/opt/data/geth")?.destinationPath;
   }

launcher/src/backend/ethereum-services/GrandineBeaconService.js

@@ -103,10 +103,6 @@ export class GrandineBeaconService extends NodeService {
     return "stereum-" + this.id + ":5054";
   }
 
-  buildPrometheusJob() {
-    return `\n  - job_name: stereum-${this.id}\n    static_configs:\n      - targets: [${this.buildConsensusClientMetricsEndpoint()}]`;
-  }
-
   getDataDir() {
     return this.volumes.find((volume) => volume.servicePath === "/opt/app/data")?.destinationPath;
   }

launcher/src/backend/ethereum-services/LighthouseBeaconService.js

@@ -107,10 +107,6 @@ export class LighthouseBeaconService extends NodeService {
     return "stereum-" + this.id + ":5054";
   }
 
-  buildPrometheusJob() {
-    return `\n  - job_name: stereum-${this.id}\n    static_configs:\n      - targets: [${this.buildConsensusClientMetricsEndpoint()}]`;
-  }
-
   getDataDir() {
     return this.volumes.find((volume) => volume.servicePath === "/opt/app/beacon")?.destinationPath;
   }

launcher/src/backend/ethereum-services/LodestarBeaconService.js

@@ -99,12 +99,6 @@ export class LodestarBeaconService extends NodeService {
     return "stereum-" + this.id + ":8008";
   }
 
-  buildPrometheusJob() {
-    return `\n  - job_name: stereum-${
-      this.id
-    }\n    metrics_path: /metrics\n    static_configs:\n      - targets: [${this.buildConsensusClientMetricsEndpoint()}]`;
-  }
-
   getDataDir() {
     return this.volumes.find((volume) => volume.servicePath === "/opt/app/beacon")?.destinationPath;
   }

launcher/src/backend/ethereum-services/NethermindService.js

@@ -80,10 +80,6 @@ export class NethermindService extends NodeService {
     return "stereum-" + this.id + ":6060";
   }
 
-  buildPrometheusJob() {
-    return `\n  - job_name: stereum-${this.id}\n    static_configs:\n      - targets: [${this.buildExecutionClientMetricsEndpoint()}]`;
-  }
-
   getDataDir() {
     return this.volumes.find((volume) => volume.servicePath === "/opt/app/data")?.destinationPath;
   }

launcher/src/backend/ethereum-services/NimbusBeaconService.js

@@ -94,10 +94,6 @@ export class NimbusBeaconService extends NodeService {
     return "stereum-" + this.id + ":8008";
   }
 
-  buildPrometheusJob() {
-    return `\n  - job_name: "nimbus"\n    metrics_path: /metrics\n    static_configs:\n      - targets: [${this.buildConsensusClientMetricsEndpoint()}]`;
-  }
-
   getDataDir() {
     return this.volumes.find((volume) => volume.servicePath === "/opt/app/beacon")?.destinationPath;
   }

launcher/src/backend/ethereum-services/OpErigonService.js

@@ -101,12 +101,6 @@ export class OpErigonService extends NodeService {
   buildExecutionClientMetricsEndpoint() {
     return "stereum-" + this.id + ":6060";
   }
-
-  buildPrometheusJob() {
-    return `\n  - job_name: stereum-${
-      this.id
-    }\n    metrics_path: /debug/metrics/prometheus\n    static_configs:\n      - targets: [${this.buildExecutionClientMetricsEndpoint()}]`;
-  }
 }
 
 // EOF

launcher/src/backend/ethereum-services/OpGethService.js

@@ -106,12 +106,6 @@ export class OpGethService extends NodeService {
   buildExecutionClientMetricsEndpoint() {
     return "stereum-" + this.id + ":6060";
   }
-
-  buildPrometheusJob() {
-    return `\n  - job_name: stereum-${
-      this.id
-    }\n    metrics_path: /debug/metrics/prometheus\n    static_configs:\n      - targets: [${this.buildExecutionClientMetricsEndpoint()}]`;
-  }
 }
 
 // EOF

launcher/src/backend/ethereum-services/OpNodeBeaconService.js

@@ -117,10 +117,6 @@ export class OpNodeBeaconService extends NodeService {
     return "stereum-" + this.id + ":7300";
   }
 
-  buildPrometheusJob() {
-    return `\n  - job_name: stereum-${this.id}\n    static_configs:\n      - targets: [${this.buildConsensusClientMetricsEndpoint()}]`;
-  }
-
   getAvailablePorts() {
     return [
       new ServicePortDefinition(9003, "tcp", "P2P connections"),

launcher/src/backend/ethereum-services/OpRethService.js

@@ -88,12 +88,6 @@ export class OpRethService extends NodeService {
   buildExecutionClientMetricsEndpoint() {
     return "stereum-" + this.id + ":6060";
   }
-
-  buildPrometheusJob() {
-    return `\n  - job_name: stereum-${
-      this.id
-    }\n    metrics_path: /debug/metrics/prometheus\n    static_configs:\n      - targets: [${this.buildExecutionClientMetricsEndpoint()}]`;
-  }
 }
 
 // EOF

launcher/src/backend/ethereum-services/PrometheusNodeExporterService.js

@@ -40,10 +40,4 @@ export class PrometheusNodeExporterService extends NodeService {
   buildPrometheusNodeExporterClientMetricsEndpoint() {
     return "stereum-" + this.id + ":9100";
   }
-
-  buildPrometheusJob() {
-    return `\n  - job_name: stereum-${
-      this.id
-    }\n    static_configs:\n      - targets: [${this.buildPrometheusNodeExporterClientMetricsEndpoint()}]`;
-  }
 }

launcher/src/backend/ethereum-services/PrysmBeaconService.js

@@ -153,10 +153,6 @@ export class PrysmBeaconService extends NodeService {
     return "stereum-" + this.id + ":8080";
   }
 
-  buildPrometheusJob() {
-    return `\n  - job_name: stereum-${this.id}\n    static_configs:\n      - targets: [${this.buildConsensusClientMetricsEndpoint()}]`;
-  }
-
   getDataDir() {
     return this.volumes.find((volume) => volume.servicePath === "/opt/app/beacon")?.destinationPath;
   }

launcher/src/backend/ethereum-services/RethService.js

@@ -77,10 +77,6 @@ export class RethService extends NodeService {
     return "stereum-" + this.id + ":6060";
   }
 
-  buildPrometheusJob() {
-    return `\n  - job_name: stereum-${this.id}\n    static_configs:\n      - targets: [${this.buildExecutionClientMetricsEndpoint()}]`;
-  }
-
   getDataDir() {
     return this.volumes.find((volume) => volume.servicePath === "/opt/data/reth")?.destinationPath;
   }

launcher/src/backend/ethereum-services/SSVNetworkService.js

@@ -95,10 +95,6 @@ SSVAPIPort: 16000`;
     return "stereum-" + this.id + ":16000";
   }
 
-  buildPrometheusJob() {
-    return `\n  - job_name: ssv\n    metrics_path: /metrics\n    static_configs:\n      - targets: [${this.buildValidatorClientMetricsEndpoint()}]\n  - job_name: ssv_health\n    metrics_path: /health\n    static_configs:\n      - targets: [${this.buildValidatorClientMetricsEndpoint()}]`;
-  }
-
   getAvailablePorts() {
     return [
       new ServicePortDefinition(13000, "tcp", "P2P connections"),

launcher/src/backend/ethereum-services/TekuBeaconService.js

@@ -105,12 +105,6 @@ export class TekuBeaconService extends NodeService {
     return "stereum-" + this.id + ":8008";
   }
 
-  buildPrometheusJob() {
-    return `\n  - job_name: stereum-${
-      this.id
-    }\n    scrape_timeout: 10s\n    metrics_path: /metrics\n    scheme: http\n    static_configs:\n      - targets: [${this.buildConsensusClientMetricsEndpoint()}]`;
-  }
-
   getDataDir() {
     return this.volumes.find((volume) => volume.servicePath === "/opt/app/data")?.destinationPath;
   }

launcher/src/backend/tests/integration/GrandineBeaconService.int.js

@@ -69,7 +69,8 @@ test("grandine consensus client", async () => {
   geth.imageVersion = versions[geth.network][geth.service].slice(-1).pop();
   grandineBC.imageVersion = versions[grandineBC.network][grandineBC.service].slice(-1).pop();
 
-  await nodeConnection.writeServiceConfiguration(geth.buildConfiguration()), await serviceManager.manageServiceState(geth.id, "started");
+  await serviceManager.manageServiceState(geth.id, "started");
+  await nodeConnection.writeServiceConfiguration(geth.buildConfiguration());
 
   //write configs for grandine BC
   await nodeConnection.writeServiceConfiguration(grandineBC.buildConfiguration());

launcher/src/backend/tests/integration/LighthouseBeaconService.int.js

@@ -76,7 +76,8 @@ test("lighthouse validator import", async () => {
   lhBC.imageVersion = versions[lhBC.network][lhBC.service].slice(-1).pop();
   lhVC.imageVersion = versions[lhVC.network][lhVC.service].slice(-1).pop();
 
-  await nodeConnection.writeServiceConfiguration(geth.buildConfiguration()), await serviceManager.manageServiceState(geth.id, "started");
+  await serviceManager.manageServiceState(geth.id, "started");
+  await nodeConnection.writeServiceConfiguration(geth.buildConfiguration());
 
   //write configs for lighhouse BC and VC
   await nodeConnection.writeServiceConfiguration(lhBC.buildConfiguration());

launcher/src/backend/tests/integration/LodestarBeaconService.int.js

@@ -77,7 +77,8 @@ test("lodestar validator import", async () => {
   lBC.imageVersion = versions[lBC.network][lBC.service].slice(-1).pop();
   lVC.imageVersion = versions[lVC.network][lVC.service].slice(-1).pop();
 
-  await nodeConnection.writeServiceConfiguration(geth.buildConfiguration()), await serviceManager.manageServiceState(geth.id, "started");
+  await serviceManager.manageServiceState(geth.id, "started");
+  await nodeConnection.writeServiceConfiguration(geth.buildConfiguration());
 
   //write configs for lodestar BC and VC
   await nodeConnection.writeServiceConfiguration(lBC.buildConfiguration());

launcher/src/backend/tests/unit/BesuService.test.js

@@ -36,11 +36,6 @@ test("buildExecutionClientMetricsEndpoint", () => {
   expect(besuMetricsEndpoint).toMatch(/stereum-.{36}:9545/);
 });
 
-test("buildPrometheusJob", () => {
-  const besuPrometheusJob = BesuService.buildByUserInput("goerli", [], "/opt/stereum/besu").buildPrometheusJob();
-  expect(besuPrometheusJob).toMatch(/\n {2}- job_name: stereum-.{36}\n {4}static_configs:\n {6}- targets: \[stereum-.{36}:9545]/);
-});
-
 test("buildByConfiguration", () => {
   const besu = BesuService.buildByConfiguration({
     id: "297",

launcher/src/backend/tests/unit/NethermindService.test.js

@@ -47,11 +47,6 @@ test("buildExecutionClientMetricsEndpoint", () => {
   expect(nethermindMetricsEndpoint).toMatch(/stereum-.{36}:6060/);
 });
 
-test("buildPrometheusJob", () => {
-  const nethermindPrometheusJob = NethermindService.buildByUserInput("goerli", [], "/opt/stereum/nethermind").buildPrometheusJob();
-  expect(nethermindPrometheusJob).toMatch(/\n {2}- job_name: stereum-.{36}\n {4}static_configs:\n {6}- targets: \[stereum-.{36}:6060]/);
-});
-
 test("buildByConfiguration", () => {
   const nethermind = NethermindService.buildByConfiguration({
     id: "297",