feat: add ssh access to test VMs and add test with dynamic cinder volume attachment

Signed-off-by: Paul Kroeher <paul.kroeher@cyberus-technology.de>

On-behalf-of: SAP paul.kroeher@sap.com

Author: pkr4711

modules/storage/cinder-storage-node.nix

@@ -9,15 +9,6 @@
 }:
 with lib;
 let
-  # adminEnv = {
-  #   OS_USERNAME = "admin";
-  #   OS_PASSWORD = "admin";
-  #   OS_PROJECT_NAME = "admin";
-  #   OS_USER_DOMAIN_NAME = "Default";
-  #   OS_PROJECT_DOMAIN_NAME = "Default";
-  #   OS_AUTH_URL = "http://controller:5000/v3";
-  #   OS_IDENTITY_API_VERSION = "3";
-  # };
   cfg = config.cinder-storage-node;
 
   cinder_env = pkgs.python3.buildEnv.override {
@@ -32,6 +23,7 @@ let
     paths = [
       cinder_env
       pkgs.qemu
+      pkgs.tgt
     ];
   };
 
@@ -52,6 +44,10 @@ let
     rootwrap_config = ${rootwrapConf}
     glance_api_servers = http://controller:9292
     verify_glance_signatures = disabled
+    log_dir = /var/log/cinder
+    iscsi_ip_address = $my_ip
+    iscsi_port = 3260
+    iscsi_target_prefix = iqn.2010-10.org.openstack:
 
     [database]
     connection = mysql+pymysql://cinder:cinder@controller/cinder
@@ -77,6 +73,9 @@ let
     lvm_type = default
     target_protocol = iscsi
     target_helper = tgtadm
+    iscsi_ip_address = $my_ip
+    iscsi_port = 3260
+    iscsi_target_prefix = iqn.2010-10.org.openstack:
   '';
 
   cinderTgtConf = pkgs.writeText "cinder.conf" ''
@@ -169,7 +168,7 @@ in
       enable = true;
       description = "iSCSI target framework daemon";
       wantedBy = [ "multi-user.target" ];
-       after = [
+      after = [
         "network.target"
         "cinder-volume-group-setup.service"
       ];
@@ -179,7 +178,7 @@ in
       ];
       environment.TGTD_CONFIG = "/etc/tgt/targets.conf";
       serviceConfig = {
-        ExecStart = "${pkgs.tgt}/bin/tgtd";
+        ExecStart = "${pkgs.tgt}/bin/tgtd -f";
         ExecStartPost = [
           "${pkgs.coreutils}/bin/sleep 5"
           "${pkgs.tgt}/bin/tgtadm --op update --mode sys --name State -v offline"

modules/testing/README.md

@@ -0,0 +1,51 @@
+# SSH Access
+
+## Storage VM
+
+```bash
+ssh root@localhost -p 2022 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+```
+
+## Controller VM
+
+```bash
+ssh root@localhost -p 1122 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+```
+
+# Cinder & tgtd
+
+## Create volume
+
+* login in to `controllerVM` or `storageVM`
+
+```bash
+openstack volume create --size 4 TEST_VOL
+```
+
+* Cinder will create a LVM logical volume in the volume group `cinder-volumes`.
+* At this time this volume will not be exposed as an iSCSI volume. This only happens when the volume is assigned to a VM.
+
+## Assign volume to a VM
+
+* create a new VM with: `openstack server create`
+  * or look into system unit: `openstack-create-vm` on VM `controllerVM`
+
+* attach volume to vm
+
+```bash
+openstack server list
+openstack volume list
+openstack server add volume <INSTANCE_NAME_OR_ID> <VOLUME_NAME_OR_ID>
+```
+
+* Verify tgtd status on `storageVM`: `tgt-admin --dump`
+* Cinder should generated a tgtd configuration file within: `/var/lib/cinder/volumes`
+* In the VM `computeVM` should appear a new block device `sda`. (`dmesg`)
+
+```bash
+[root@storageVM:/var/lib/cinder/volumes]# l
+total 12K
+drwxr-xr-x 2 cinder cinder 4.0K Feb 24 08:16 .
+drwxr-xr-x 6 cinder cinder 4.0K Feb 24 08:09 ..
+-rw------- 1 cinder cinder  268 Feb 24 08:16 volume-64159e0c-18bb-449f-b1e0-86f198b170e4
+```

modules/testing/default.nix

@@ -33,9 +33,26 @@ let
 
         environment.systemPackages = [
           pkgs.openstackclient
+          pkgs.openiscsi
+          pkgs.sshpass
         ];
 
         environment.variables = adminEnv;
+
+        # enable easy access to test VMs with ssh
+        users.users.root.openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEMjGRp1/vaTxGiGbZnaSv4wu4LUUP7lGSGDFKfF31xw paul.kroeher@cyberus-technology.de"
+        ];
+
+        services.openssh = {
+          enable = true;
+          ports = [ 22 ];
+          settings = {
+            PasswordAuthentication = true;
+            PermitRootLogin = "without-password"; # "yes", "without-password", "prohibit-password", "forced-commands-only", "no"
+          };
+        };
+
       };
     };
 in
@@ -66,6 +83,10 @@ in
             vlan = 2;
           };
         };
+        # enable ssh access
+        forwardPorts = [
+         { from = "host"; host.port = 1122; guest.port = 22; }
+        ];
       };
 
       systemd.services.openstack-create-vm = {
@@ -121,14 +142,13 @@ in
             matchConfig.Name = [ "eth0" ];
             networkConfig = {
               DHCP = "yes";
+              DNS = "8.8.8.8";
             };
           };
           eth1 = {
             matchConfig.Name = [ "eth1" ];
             networkConfig = {
               Address = "10.0.0.11/24";
-              Gateway = "10.0.0.1";
-              DNS = "8.8.8.8";
             };
           };
 
@@ -201,7 +221,7 @@ in
         diskSize = 4096;
         # add separate disk as LVM backend
         emptyDiskImages = [
-          16384
+          16384 # 16GB
         ];
         interfaces = {
           eth1 = {
@@ -211,19 +231,31 @@ in
             vlan = 2;
           };
         };
+        # enable ssh access
+        forwardPorts = [
+         { from = "host"; host.port = 2022; guest.port = 22; }
+        ];
       };
 
       systemd.network = {
         enable = true;
         wait-online.enable = false;
 
         networks = {
+          eth0 = {
+            matchConfig.Name = [ "eth0" ];
+            networkConfig = {
+              DHCP = "yes";
+              LinkLocalAddressing = "yes";
+              KeepConfiguration = "yes";
+              DNS = "8.8.8.8";
+            };
+          };
+
           eth1 = {
             matchConfig.Name = [ "eth1" ];
             networkConfig = {
               Address = "10.0.0.20/24";
-              Gateway = "10.0.0.1";
-              DNS = "8.8.8.8";
             };
           };
 

tests/openstack-default-setup.nix

@@ -64,7 +64,11 @@ pkgs.nixosTest {
           if status != 0:
             continue
           compute_nodes = json.loads(out)
-          if len(net_agents) == 4 and len(compute_nodes) == 1 and compute_nodes[0].get("Host","None") == "computeVM":
+          status, out = controllerVM.execute("openstack volume service list -f json")
+          if status != 0:
+            continue
+          storage_nodes = json.loads(out)
+          if len(storage_nodes) == 2 and len(net_agents) == 4 and len(compute_nodes) == 1 and compute_nodes[0].get("Host","None") == "computeVM":
             return True
         return False
 
@@ -107,6 +111,10 @@ pkgs.nixosTest {
       controllerVM.wait_for_unit("nova-scheduler.service")
       controllerVM.wait_for_unit("nova-conductor.service")
 
+      # check all services on storageVM
+      storageVM.wait_for_unit("tgtd.service")
+      storageVM.wait_for_unit("cinder-volume.service")
+
       assert wait_for_openstack()
 
       controllerVM.succeed("systemctl start nova-host-discovery.service")
@@ -126,5 +134,19 @@ pkgs.nixosTest {
       # Ping the OpenStack VM from the controller host. We use the network
       # namespace dedicated for the VM to ping it.
       assert retry_until_succeed(controllerVM, f"ip netns exec {net_ns} ping -c 1 {vm_ip}", 30)
+
+      # create volume with 4GB
+      controllerVM.execute("openstack volume create --size 4 test_vol")
+      # attach volume to VM
+      controllerVM.execute("openstack server add volume test_vm test_vol")
+      # wait until volume is attached
+      time.sleep(20)
+
+      retry_until_succeed(controllerVM, f"ip netns exec {net_ns} sshpass -p gocubsgo ssh cirros@{vm_ip} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null lsblk", 60)
+      # check second block device of VM
+      status, output = controllerVM.execute(f"ip netns exec {net_ns} sshpass -p gocubsgo ssh cirros@{vm_ip} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null lsblk | grep vdb")
+      output = output.strip()
+      assert status == 0
+      assert output == "vdb     252:16   0    4G  0 disk"
     '';
 }