xxxx: create separate storage node

Signed-off-by: Paul Kroeher <paul.kroeher@cyberus-technology.de>

Author: pkr4711

modules/compute/nova.nix

@@ -184,14 +184,7 @@ in
             user = "nova";
           };
         };
-        "/etc/systemd/system/tgtd.service" = {
-          "C+" = {
-            user = "root";
-            group = "root";
-            mode = "0600";
-            argument = "${pkgs.tgt}/etc/systemd/system/tgtd.service";
-          };
-        };
+        # we don't need tgt on a compute node -> only iscsi-client (openiscsi)
       };
     };
 
@@ -200,7 +193,7 @@ in
       name = "iqn.iscsi.${config.networking.hostName}";
     };
 
-    environment.systemPackages = [ pkgs.tgt ];
+    environment.systemPackages = [ pkgs.openiscsi ];
 
     systemd.services.nova-compute = {
       description = "OpenStack Nova Scheduler Daemon";
@@ -217,7 +210,7 @@ in
           qemu
           util-linux
           lvm2
-          tgt
+          openiscsi
         ]
         ++ cfg.extraPkgs;
       environment.PYTHONPATH = "${nova_env}/${pkgs.python3.sitePackages}";

modules/controller/openstack-controller.nix

@@ -33,8 +33,9 @@ in
     (import ./nova.nix { inherit nova; })
     (import ./neutron.nix { inherit neutron; })
     (import ./horizon.nix { inherit horizon; })
-    (import ./cinder.nix { inherit cinder; })
-    (import ../storage/cinder-storage-node.nix { inherit cinder; })
+    (import ./cinder.nix { inherit cinder; }) # only cinder management component
+    # deploy real storage backend into a separate storageNode
+    # (import ../storage/cinder-storage-node.nix { inherit cinder; })
   ];
 
   config = {

modules/default.nix

@@ -14,5 +14,7 @@
 
   computeModule = import ./compute/compute.nix { inherit (openstackPkgs) neutron nova; };
 
+  storageModule = import ./storage/cinder-storage-node.nix { inherit (openstackPkgs) cinder; };
+
   testModules = import ./testing { };
 }

modules/storage/cinder-storage-node.nix

@@ -45,7 +45,7 @@ let
     [DEFAULT]
     transport_url = rabbit://openstack:openstack@controller
     auth_strategy = keystone
-    my_ip = controller
+    my_ip = 10.0.0.20
     enabled_backends = lvm
     volumes_dir = /var/lib/cinder/volumes
     state_path = /var/lib/cinder
@@ -76,6 +76,11 @@ let
     volume_backend_name = lvm
     lvm_type = default
     target_protocol = iscsi
+    target_helper = tgtadm
+  '';
+
+  cinderTgtConf = pkgs.writeText "cinder.conf" ''
+    include /var/lib/cinder/volumes/*
   '';
 in
 {
@@ -140,10 +145,60 @@ in
             mode = "0755";
           };
         };
+        "/etc/cinder/cinder.conf" = {
+          L = {
+            argument = "${cinderConf}";
+          };
+        };
+        "/etc/tgt/conf.d/cinder.conf" = {
+          L = {
+            argument = "${cinderTgtConf}";
+          };
+        };
+        "/etc/tgt/targets.conf" = {
+          L = {
+            argument = "${pkgs.tgt}/etc/tgt/targets.conf";
+          };
+        };
+      };
+    };
+
+    # start iSCSI target daemon
+    # we expose LVM block storage as iSCSI to compute hosts
+    systemd.services.tgtd = {
+      enable = true;
+      description = "iSCSI target framework daemon";
+      wantedBy = [ "multi-user.target" ];
+       after = [
+        "network.target"
+        "cinder-volume-group-setup.service"
+      ];
+      path = [
+        pkgs.coreutils
+        pkgs.tgt
+      ];
+      environment.TGTD_CONFIG = "/etc/tgt/targets.conf";
+      serviceConfig = {
+        ExecStart = "${pkgs.tgt}/bin/tgtd";
+        ExecStartPost = [
+          "${pkgs.coreutils}/bin/sleep 5"
+          "${pkgs.tgt}/bin/tgtadm --op update --mode sys --name State -v offline"
+          "${pkgs.tgt}/bin/tgtadm --op update --mode sys --name State -v ready"
+          "${pkgs.tgt}/bin/tgt-admin -e -c $TGTD_CONFIG"
+        ];
+
+        ExecReload = "${pkgs.tgt}/bin/tgt-admin --update ALL -f -c $TGTD_CONFIG";
+
+        ExecStop = [
+          "${pkgs.tgt}/bin/tgtadm --op update --mode sys --name State -v offline"
+          "${pkgs.tgt}/bin/tgt-admin --offline ALL"
+          "${pkgs.tgt}/bin/tgt-admin --update ALL -c /dev/null -f"
+          "${pkgs.tgt}/bin/tgtadm --op delete --mode system"
+        ];
       };
     };
 
-    systemd.services.cinder-volume-group = {
+    systemd.services.cinder-volume-group-setup = {
       description = "OpenStack Cinder volume group setup";
       wantedBy = [ "multi-user.target" ];
       path = [
@@ -155,14 +210,9 @@ in
         ExecStart = pkgs.writeShellScript "cinder-volume-group.sh" ''
           set -euxo pipefail
 
-          # Setup some lvm volume group required by cinder
-          dd if=/dev/zero of=/tmp/cinder-volumes bs=1G count=2
-
-          losetup /dev/loop0 /tmp/cinder-volumes
-
-          # Create physical volume and volume group
-          pvcreate /dev/loop0
-          vgcreate cinder-volumes /dev/loop0
+          # create a new LVM volume group on second disk
+          pvcreate /dev/vdb
+          vgcreate cinder-volumes /dev/vdb
         '';
       };
     };
@@ -173,12 +223,13 @@ in
     # Update: still does not work -.-
     environment.systemPackages = [
       pkgs.qemu
+      pkgs.tgt
     ];
 
     systemd.services.cinder-volume = {
       description = "OpenStack Cinder Volume";
       after = [
-        "cinder-volume-group.service"
+        "cinder-volume-group-setup.service"
       ];
       path = with pkgs; [
         cinder_env

modules/testing/default.nix

@@ -188,4 +188,54 @@ in
       };
 
     };
+
+  testStorage =
+    { ... }:
+    {
+
+      imports = [ common ];
+
+      virtualisation = {
+        memorySize = 4096;
+        cores = 4;
+        diskSize = 4096;
+        # add separate disk as LVM backend
+        emptyDiskImages = [
+          16384
+        ];
+        interfaces = {
+          eth1 = {
+            vlan = 1;
+          };
+          eth2 = {
+            vlan = 2;
+          };
+        };
+      };
+
+      systemd.network = {
+        enable = true;
+        wait-online.enable = false;
+
+        networks = {
+          eth1 = {
+            matchConfig.Name = [ "eth1" ];
+            networkConfig = {
+              Address = "10.0.0.20/24";
+              Gateway = "10.0.0.1";
+              DNS = "8.8.8.8";
+            };
+          };
+
+          eth2 = {
+            matchConfig.Name = [ "eth2" ];
+            networkConfig = {
+              DHCP = "no";
+              LinkLocalAddressing = "no";
+              KeepConfiguration = "yes";
+            };
+          };
+        };
+      };
+    };
 }

tests/openstack-default-setup.nix

@@ -23,6 +23,15 @@ pkgs.nixosTest {
       ];
     };
 
+  nodes.storageVM =
+    { ... }:
+    {
+      imports = [
+        nixosModules.storageModule
+        nixosModules.testModules.testStorage
+      ];
+    };
+
   testScript =
     { ... }:
     ''