FIX: Integration and Molecule Tests

.github/workflows/test-molecule.yml

@@ -68,9 +68,9 @@ jobs:
       matrix:
         tests:
           [
-            { role: "update-changes", test: "222" },
-            { role: "update-changes", test: "223" },
-            { role: "update-changes", test: "224" },
+            { role: "update-changes", test: "232" },
+            { role: "update-changes", test: "235" },
+            { role: "update-changes", test: "241" },
           ]
       fail-fast: false
     concurrency: molecule-test-${{ matrix.tests.role }}-${{ matrix.tests.test }}

controls/defaults/stereum_defaults.yaml

@@ -7,6 +7,9 @@ stereum_static:
       lane: stable
       unattended:
         install: false
+        interval_days: 1
+        hour: 1
+        min: 30
     arch: x86_64
     versions:
       # consensus clients

controls/roles/manage-service/tasks/main.yml

@@ -44,7 +44,7 @@
         owner: "{{ stereum_service_configuration.user }}"
         group: "{{ stereum_service_configuration.user }}"
         mode: 0700
-        recurse: yes
+        recurse: "{{ 'no' if (stereum_service_configuration.service == 'NimbusValidatorService' and item.split(':') | last == '/opt/app/secrets') else 'yes' }}"
       changed_when: false
       become: yes
 
@@ -53,7 +53,6 @@
         - item.split(':') | first != '/'
         - item.split(':') | first != '/sys'
         - item.split(':') | first != '/proc'
-        - not (stereum_service_configuration.service == "NimbusValidatorService" and item.split(':') | last == '/opt/app/secrets') # ignoring this bc nimbus validator sets this directory to 0600
       with_items: "{{ stereum_service_configuration.volumes | reject('search', ':/engine.jwt|:/execution/engine.jwt|:/op-engine.jwt') }}"
 
     - name: Set directory permissions for engine.jwt

launcher/src/backend/tests/integration/BesuService.int.js

@@ -56,7 +56,7 @@ test("besu installation", async () => {
   await nodeConnection.prepareStereumNode(nodeConnection.settings.stereum.settings.controls_install_path);
 
   //install besu
-  let executionClient = serviceManager.getService("BesuService", { network: "holesky", installDir: "/opt/stereum" });
+  let executionClient = serviceManager.getService("BesuService", { network: "hoodi", installDir: "/opt/stereum" });
 
   let versions = await nodeConnection.nodeUpdates.checkUpdates();
   executionClient.imageVersion = versions[executionClient.network][executionClient.service].slice(-1).pop();

launcher/src/backend/tests/integration/ErigonService.int.js

@@ -56,7 +56,7 @@ test("erigon installation", async () => {
   await nodeConnection.prepareStereumNode(nodeConnection.settings.stereum.settings.controls_install_path);
 
   //install erigon
-  let executionClient = serviceManager.getService("ErigonService", { network: "holesky", installDir: "/opt/stereum" });
+  let executionClient = serviceManager.getService("ErigonService", { network: "hoodi", installDir: "/opt/stereum" });
 
   let versions = await nodeConnection.nodeUpdates.checkUpdates();
   executionClient.imageVersion = versions[executionClient.network][executionClient.service].slice(-1).pop();

launcher/src/backend/tests/integration/FlashbotsMevBoostService.int.js

@@ -57,9 +57,9 @@ test("mevboost installation", async () => {
 
   //install mevboost
   let mevboost = serviceManager.getService("FlashbotsMevBoostService", {
-    network: "holesky",
+    network: "hoodi",
     relays:
-      "https://0xafa4c6985aa049fb79dd37010438cfebeb0f2bd42b115b89dd678dab0670c1de38da0c4e9138c9290a398ecd9a0b3110@boost-relay-holesky.flashbots.net",
+      "https://0xafa4c6985aa049fb79dd37010438cfebeb0f2bd42b115b89dd678dab0670c1de38da0c4e9138c9290a398ecd9a0b3110@boost-relay-hoodi.flashbots.net",
   });
 
   let versions = await nodeConnection.nodeUpdates.checkUpdates();
@@ -76,7 +76,7 @@ test("mevboost installation", async () => {
     await testServer.Sleep(30000);
     status = await nodeConnection.sshService.exec(`docker logs stereum-${mevboost.id}`);
     if (
-      /Listening on 0.0.0.0:18550/.test(status.stdout) &&
+      /listening on 0.0.0.0:18550/.test(status.stdout) &&
       /using 1 relays/.test(status.stdout) &&
       !/Invalid relay URL/.test(status.stdout)
     ) {
@@ -97,7 +97,7 @@ test("mevboost installation", async () => {
   }
 
   // check if Mevboost service is running properly
-  expect(status.stdout).toMatch(/Listening on 0.0.0.0:18550/);
+  expect(status.stdout).toMatch(/listening on 0.0.0.0:18550/);
   expect(status.stdout).toMatch(/using 1 relays/);
   expect(status.stdout).not.toMatch(/Invalid relay URL/);
 });

launcher/src/backend/tests/integration/GethService.int.js

@@ -7,7 +7,7 @@ import { ServiceManager } from "../../ServiceManager.js";
 import { TaskManager } from "../../TaskManager.js";
 const log = require("electron-log");
 
-jest.setTimeout(500000);
+jest.setTimeout(600000);
 
 test("geth installation", async () => {
   const testServer = new HetznerServer();
@@ -56,7 +56,7 @@ test("geth installation", async () => {
   await nodeConnection.prepareStereumNode(nodeConnection.settings.stereum.settings.controls_install_path);
 
   //install geth
-  let executionClient = serviceManager.getService("GethService", { network: "holesky", installDir: "/opt/stereum" });
+  let executionClient = serviceManager.getService("GethService", { network: "hoodi", installDir: "/opt/stereum" });
 
   let versions = await nodeConnection.nodeUpdates.checkUpdates();
   executionClient.imageVersion = versions[executionClient.network][executionClient.service].slice(-1).pop();
@@ -76,7 +76,6 @@ test("geth installation", async () => {
       /Started P2P networking/.test(status.stderr) &&
       /Starting metrics server/.test(status.stderr) &&
       /Loaded JWT secret file/.test(status.stderr) &&
-      /Looking for peers/.test(status.stderr) &&
       /HTTP server started/.test(status.stderr)
     ) {
       condition = true;
@@ -106,6 +105,5 @@ test("geth installation", async () => {
   expect(status.stderr).toMatch(/Started P2P networking/);
   expect(status.stderr).toMatch(/Starting metrics server/);
   expect(status.stderr).toMatch(/Loaded JWT secret file/);
-  expect(status.stderr).toMatch(/Looking for peers/);
   expect(status.stderr).toMatch(/HTTP server started/);
 });

launcher/src/backend/tests/integration/GrandineBeaconService.int.js

@@ -56,10 +56,10 @@ test("grandine consensus client", async () => {
   await nodeConnection.findStereumSettings();
   await nodeConnection.prepareStereumNode(nodeConnection.settings.stereum.settings.controls_install_path);
 
-  let geth = serviceManager.getService("GethService", { network: "holesky", installDir: "/opt/stereum" });
+  let geth = serviceManager.getService("GethService", { network: "hoodi", installDir: "/opt/stereum" });
 
   let grandineBC = serviceManager.getService("GrandineBeaconService", {
-    network: "holesky",
+    network: "hoodi",
     installDir: "/opt/stereum",
     executionClients: [geth],
   });

launcher/src/backend/tests/integration/L2GethService.int.js

@@ -58,8 +58,9 @@ test("l2geth installation", async () => {
   //install besu
   let executionClient = serviceManager.getService("L2GethService", { network: "op-mainnet", installDir: "/opt/stereum" });
 
-  let versions = await nodeConnection.nodeUpdates.checkUpdates();
-  executionClient.imageVersion = versions[executionClient.network][executionClient.service].slice(-1).pop();
+  // let versions = await nodeConnection.nodeUpdates.checkUpdates();
+  // executionClient.imageVersion = versions[executionClient.network][executionClient.service].slice(-1).pop();
+  executionClient.imageVersion = "latest";
 
   await nodeConnection.writeServiceConfiguration(executionClient.buildConfiguration());
   await serviceManager.manageServiceState(executionClient.id, "started");
@@ -68,19 +69,22 @@ test("l2geth installation", async () => {
   let condition = false;
   let counter = 0;
   let status = "";
-  while (!condition && counter < 10) {
+  let completeStatus = "";
+  while (!condition && counter < 5) {
     await testServer.Sleep(30000);
     status = await nodeConnection.sshService.exec(`docker logs stereum-${executionClient.id}`);
+    completeStatus = status.stdout + status.stderr;
+
     if (
-      /Starting peer-to-peer node/.test(status.stdout) &&
-      /Opened ancient database/.test(status.stdout) &&
-      /Initialised chain configuration/.test(status.stdout) &&
-      /Loaded most recent local header/.test(status.stdout) &&
-      /Loaded most recent local full block/.test(status.stdout) &&
-      /Loaded most recent local fast block/.test(status.stdout) &&
-      /Started P2P networking/.test(status.stdout) &&
-      /IPC endpoint opened/.test(status.stdout) &&
-      /HTTP endpoint opened/.test(status.stdout)
+      /Starting peer-to-peer node/.test(completeStatus) &&
+      /Opened ancient database/.test(completeStatus) &&
+      /Initialised chain configuration/.test(completeStatus) &&
+      /Loaded most recent local header/.test(completeStatus) &&
+      /Loaded most recent local full block/.test(completeStatus) &&
+      /Loaded most recent local fast block/.test(completeStatus) &&
+      /Started P2P networking/.test(completeStatus) &&
+      /IPC endpoint opened/.test(completeStatus) &&
+      /HTTP endpoint opened/.test(completeStatus)
     ) {
       condition = true;
     }
@@ -103,13 +107,13 @@ test("l2geth installation", async () => {
     expect((docker.stdout.match(new RegExp("Up", "g")) || []).length).toBe(1);
   }
 
-  expect(status.stdout).toMatch(/Starting peer-to-peer node/);
-  expect(status.stdout).toMatch(/Opened ancient database/);
-  expect(status.stdout).toMatch(/Initialised chain configuration/);
-  expect(status.stdout).toMatch(/Loaded most recent local header/);
-  expect(status.stdout).toMatch(/Loaded most recent local full block/);
-  expect(status.stdout).toMatch(/Loaded most recent local fast block/);
-  expect(status.stdout).toMatch(/Started P2P networking/);
-  expect(status.stdout).toMatch(/IPC endpoint opened/);
-  expect(status.stdout).toMatch(/HTTP endpoint opened/);
+  expect(completeStatus).toMatch(/Starting peer-to-peer node/);
+  expect(completeStatus).toMatch(/Opened ancient database/);
+  expect(completeStatus).toMatch(/Initialised chain configuration/);
+  expect(completeStatus).toMatch(/Loaded most recent local header/);
+  expect(completeStatus).toMatch(/Loaded most recent local full block/);
+  expect(completeStatus).toMatch(/Loaded most recent local fast block/);
+  expect(completeStatus).toMatch(/Started P2P networking/);
+  expect(completeStatus).toMatch(/IPC endpoint opened/);
+  expect(completeStatus).toMatch(/HTTP endpoint opened/);
 });

launcher/src/backend/tests/integration/LighthouseBeaconService.int.js

@@ -56,16 +56,16 @@ test("lighthouse validator import", async () => {
   await nodeConnection.findStereumSettings();
   await nodeConnection.prepareStereumNode(nodeConnection.settings.stereum.settings.controls_install_path);
 
-  let geth = serviceManager.getService("GethService", { network: "holesky", installDir: "/opt/stereum" });
+  let geth = serviceManager.getService("GethService", { network: "hoodi", installDir: "/opt/stereum" });
 
   let lhBC = serviceManager.getService("LighthouseBeaconService", {
-    network: "holesky",
+    network: "hoodi",
     installDir: "/opt/stereum",
     executionClients: [geth],
   });
 
   let lhVC = serviceManager.getService("LighthouseValidatorService", {
-    network: "holesky",
+    network: "hoodi",
     installDir: "/opt/stereum",
     consensusClients: [lhBC],
   });
@@ -99,7 +99,8 @@ test("lighthouse validator import", async () => {
     ],
     passwords: ["MyTestPassword", "MyTestPassword", "MyTestPassword"],
   });
-  await validatorAccountManager.importKey(lhVC.id);
+  const importResult = await validatorAccountManager.importKey(lhVC.id);
+  log.info(importResult);
 
   //get logs
   let condition = false;
@@ -116,15 +117,14 @@ test("lighthouse validator import", async () => {
       /HTTP API started/.test(BCstatus.stderr) &&
       /Metrics HTTP server started/.test(BCstatus.stderr) &&
       /Syncing /.test(BCstatus.stderr) &&
-      /The execution endpoint is connected and configured, however it is not yet synced/.test(BCstatus.stderr) &&
+      /Execution endpoint is not synced/.test(BCstatus.stderr) &&
       /Successfully loaded graffiti file/.test(VCstatus.stderr) &&
       /Starting validator client/.test(VCstatus.stderr) &&
       /Metrics HTTP server started/.test(VCstatus.stderr) &&
       /Successfully loaded graffiti file/.test(VCstatus.stderr) &&
       /Initialized beacon node connections/.test(VCstatus.stderr) &&
       /HTTP API started/.test(VCstatus.stderr) &&
-      /Imported keystores via standard HTTP API, count: 3/.test(VCstatus.stderr) &&
-      /Enabled validator/.test(VCstatus.stderr)
+      /Imported keystores via standard HTTP API, count: 3/.test(VCstatus.stderr)
     ) {
       condition = true;
     }
@@ -162,7 +162,7 @@ test("lighthouse validator import", async () => {
   expect(BCstatus.stderr).toMatch(/HTTP API started/);
   expect(BCstatus.stderr).toMatch(/Metrics HTTP server started/);
   expect(BCstatus.stderr).toMatch(/Syncing/);
-  expect(BCstatus.stderr).toMatch(/The execution endpoint is connected and configured, however it is not yet synced/);
+  expect(BCstatus.stderr).toMatch(/Execution endpoint is not synced/);
 
   //check lighthouse VC logs
   expect(VCstatus.stderr).toMatch(/Successfully loaded graffiti file/);
@@ -172,5 +172,4 @@ test("lighthouse validator import", async () => {
   expect(VCstatus.stderr).toMatch(/Initialized beacon node connections/);
   expect(VCstatus.stderr).toMatch(/HTTP API started/);
   expect(VCstatus.stderr).toMatch(/Imported keystores via standard HTTP API, count: 3/);
-  expect(VCstatus.stderr).toMatch(/Enabled validator/);
 });

launcher/src/backend/tests/integration/LodestarBeaconService.int.js

@@ -57,16 +57,16 @@ test("lodestar validator import", async () => {
   await nodeConnection.findStereumSettings();
   await nodeConnection.prepareStereumNode(nodeConnection.settings.stereum.settings.controls_install_path);
 
-  let geth = serviceManager.getService("GethService", { network: "holesky", installDir: "/opt/stereum" });
+  let geth = serviceManager.getService("GethService", { network: "hoodi", installDir: "/opt/stereum" });
 
   let lBC = serviceManager.getService("LodestarBeaconService", {
-    network: "holesky",
+    network: "hoodi",
     installDir: "/opt/stereum",
     executionClients: [geth],
   });
 
   let lVC = serviceManager.getService("LodestarValidatorService", {
-    network: "holesky",
+    network: "hoodi",
     installDir: "/opt/stereum",
     consensusClients: [lBC],
   });
@@ -100,7 +100,8 @@ test("lodestar validator import", async () => {
     ],
     passwords: ["MyTestPassword", "MyTestPassword", "MyTestPassword"],
   });
-  await validatorAccountManager.importKey(lVC.id);
+  const importResult = await validatorAccountManager.importKey(lVC.id);
+  log.info(importResult);
 
   //get logs
   let condition = false;

launcher/src/backend/tests/integration/NethermindService.int.js

@@ -56,7 +56,7 @@ test("nethermind installationm", async () => {
   await nodeConnection.prepareStereumNode(nodeConnection.settings.stereum.settings.controls_install_path);
 
   //install nethermind
-  let executionClient = serviceManager.getService("NethermindService", { network: "holesky", installDir: "/opt/stereum" });
+  let executionClient = serviceManager.getService("NethermindService", { network: "hoodi", installDir: "/opt/stereum" });
 
   let versions = await nodeConnection.nodeUpdates.checkUpdates();
   executionClient.imageVersion = versions[executionClient.network][executionClient.service].slice(-1).pop();

launcher/src/backend/tests/integration/NimbusBeaconService.int.js

@@ -58,17 +58,17 @@ test("nimbus validator import", async () => {
   await nodeConnection.prepareStereumNode(nodeConnection.settings.stereum.settings.controls_install_path);
 
   //install geth
-  let geth = serviceManager.getService("GethService", { network: "holesky", installDir: "/opt/stereum" });
+  let geth = serviceManager.getService("GethService", { network: "hoodi", installDir: "/opt/stereum" });
 
   //install nimbus
   let nimbusBC = serviceManager.getService("NimbusBeaconService", {
-    network: "holesky",
+    network: "hoodi",
     installDir: "/opt/stereum",
     executionClients: [geth],
   });
 
   let nimbusVC = serviceManager.getService("NimbusValidatorService", {
-    network: "holesky",
+    network: "hoodi",
     installDir: "/opt/stereum",
     consensusClients: [nimbusBC],
   });
@@ -101,7 +101,8 @@ test("nimbus validator import", async () => {
     ],
     passwords: ["MyTestPassword", "MyTestPassword", "MyTestPassword"],
   });
-  await validatorAccountManager.importKey(nimbusVC.id);
+  const importResult = await validatorAccountManager.importKey(nimbusVC.id);
+  log.info(importResult);
 
   //get logs
   let condition = false;