Skip to content

Commit 5c9b132

Browse files
pkr4711pkr4711
committed
xxxx: create separate storage node
Signed-off-by: Paul Kroeher <paul.kroeher@cyberus-technology.de> On-behalf-of: SAP paul.kroeher@sap.com
1 parent a4d153e commit 5c9b132

7 files changed

Lines changed: 245 additions & 35 deletions

File tree

‎modules/compute/nova.nix‎

Lines changed: 3 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -189,14 +189,7 @@ in
189189
user = "nova";
190190
};
191191
};
192-
"/etc/systemd/system/tgtd.service" = {
193-
"C+" = {
194-
user = "root";
195-
group = "root";
196-
mode = "0600";
197-
argument = "${pkgs.tgt}/etc/systemd/system/tgtd.service";
198-
};
199-
};
192+
# we don't need tgt on a compute node -> only iscsi-client (openiscsi)
200193
};
201194
};
202195

@@ -205,7 +198,7 @@ in
205198
name = "iqn.iscsi.${config.networking.hostName}";
206199
};
207200

208-
environment.systemPackages = [ pkgs.tgt ];
201+
environment.systemPackages = [ pkgs.openiscsi ];
209202

210203
systemd.services.nova-compute = {
211204
description = "OpenStack Nova Scheduler Daemon";
@@ -222,7 +215,7 @@ in
222215
qemu
223216
util-linux
224217
lvm2
225-
tgt
218+
openiscsi
226219
]
227220
++ cfg.extraPkgs;
228221
environment.PYTHONPATH = "${nova_env}/${pkgs.python3.sitePackages}";

‎modules/controller/openstack-controller.nix‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,9 @@ in
3333
(import ./nova.nix { inherit nova; })
3434
(import ./neutron.nix { inherit neutron; })
3535
(import ./horizon.nix { inherit horizon; })
36-
(import ./cinder.nix { inherit cinder; })
37-
(import ../storage/cinder-storage-node.nix { inherit cinder; })
36+
(import ./cinder.nix { inherit cinder; }) # only cinder management component
37+
# deploy real storage backend into a separate storageNode
38+
# (import ../storage/cinder-storage-node.nix { inherit cinder; })
3839
];
3940

4041
config = {

‎modules/default.nix‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,5 +14,7 @@
1414

1515
computeModule = import ./compute/compute.nix { inherit (openstackPkgs) neutron nova; };
1616

17+
storageModule = import ./storage/cinder-storage-node.nix { inherit (openstackPkgs) cinder; };
18+
1719
testModules = import ./testing { };
1820
}

‎modules/storage/cinder-storage-node.nix‎

Lines changed: 70 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -9,15 +9,6 @@
99
}:
1010
with lib;
1111
let
12-
# adminEnv = {
13-
# OS_USERNAME = "admin";
14-
# OS_PASSWORD = "admin";
15-
# OS_PROJECT_NAME = "admin";
16-
# OS_USER_DOMAIN_NAME = "Default";
17-
# OS_PROJECT_DOMAIN_NAME = "Default";
18-
# OS_AUTH_URL = "http://controller:5000/v3";
19-
# OS_IDENTITY_API_VERSION = "3";
20-
# };
2112
cfg = config.cinder-storage-node;
2213

2314
cinder_env = pkgs.python3.buildEnv.override {
@@ -32,6 +23,7 @@ let
3223
paths = [
3324
cinder_env
3425
pkgs.qemu
26+
pkgs.tgt
3527
];
3628
};
3729

@@ -45,13 +37,17 @@ let
4537
[DEFAULT]
4638
transport_url = rabbit://openstack:openstack@controller
4739
auth_strategy = keystone
48-
my_ip = controller
40+
my_ip = 10.0.0.20
4941
enabled_backends = lvm
5042
volumes_dir = /var/lib/cinder/volumes
5143
state_path = /var/lib/cinder
5244
rootwrap_config = ${rootwrapConf}
5345
glance_api_servers = http://controller:9292
5446
verify_glance_signatures = disabled
47+
log_dir = /var/log/cinder
48+
iscsi_ip_address = $my_ip
49+
iscsi_port = 3260
50+
iscsi_target_prefix = iqn.2010-10.org.openstack:
5551
5652
[database]
5753
connection = mysql+pymysql://cinder:cinder@controller/cinder
@@ -76,6 +72,14 @@ let
7672
volume_backend_name = lvm
7773
lvm_type = default
7874
target_protocol = iscsi
75+
target_helper = tgtadm
76+
iscsi_ip_address = $my_ip
77+
iscsi_port = 3260
78+
iscsi_target_prefix = iqn.2010-10.org.openstack:
79+
'';
80+
81+
cinderTgtConf = pkgs.writeText "cinder.conf" ''
82+
include /var/lib/cinder/volumes/*
7983
'';
8084
in
8185
{
@@ -140,10 +144,60 @@ in
140144
mode = "0755";
141145
};
142146
};
147+
"/etc/cinder/cinder.conf" = {
148+
L = {
149+
argument = "${cinderConf}";
150+
};
151+
};
152+
"/etc/tgt/conf.d/cinder.conf" = {
153+
L = {
154+
argument = "${cinderTgtConf}";
155+
};
156+
};
157+
"/etc/tgt/targets.conf" = {
158+
L = {
159+
argument = "${pkgs.tgt}/etc/tgt/targets.conf";
160+
};
161+
};
143162
};
144163
};
145164

146-
systemd.services.cinder-volume-group = {
165+
# start iSCSI target daemon
166+
# we expose LVM block storage as iSCSI to compute hosts
167+
systemd.services.tgtd = {
168+
enable = true;
169+
description = "iSCSI target framework daemon";
170+
wantedBy = [ "multi-user.target" ];
171+
after = [
172+
"network.target"
173+
"cinder-volume-group-setup.service"
174+
];
175+
path = [
176+
pkgs.coreutils
177+
pkgs.tgt
178+
];
179+
environment.TGTD_CONFIG = "/etc/tgt/targets.conf";
180+
serviceConfig = {
181+
ExecStart = "${pkgs.tgt}/bin/tgtd -f";
182+
ExecStartPost = [
183+
"${pkgs.coreutils}/bin/sleep 5"
184+
"${pkgs.tgt}/bin/tgtadm --op update --mode sys --name State -v offline"
185+
"${pkgs.tgt}/bin/tgtadm --op update --mode sys --name State -v ready"
186+
"${pkgs.tgt}/bin/tgt-admin -e -c $TGTD_CONFIG"
187+
];
188+
189+
ExecReload = "${pkgs.tgt}/bin/tgt-admin --update ALL -f -c $TGTD_CONFIG";
190+
191+
ExecStop = [
192+
"${pkgs.tgt}/bin/tgtadm --op update --mode sys --name State -v offline"
193+
"${pkgs.tgt}/bin/tgt-admin --offline ALL"
194+
"${pkgs.tgt}/bin/tgt-admin --update ALL -c /dev/null -f"
195+
"${pkgs.tgt}/bin/tgtadm --op delete --mode system"
196+
];
197+
};
198+
};
199+
200+
systemd.services.cinder-volume-group-setup = {
147201
description = "OpenStack Cinder volume group setup";
148202
wantedBy = [ "multi-user.target" ];
149203
path = [
@@ -155,14 +209,9 @@ in
155209
ExecStart = pkgs.writeShellScript "cinder-volume-group.sh" ''
156210
set -euxo pipefail
157211
158-
# Setup some lvm volume group required by cinder
159-
dd if=/dev/zero of=/tmp/cinder-volumes bs=1G count=2
160-
161-
losetup /dev/loop0 /tmp/cinder-volumes
162-
163-
# Create physical volume and volume group
164-
pvcreate /dev/loop0
165-
vgcreate cinder-volumes /dev/loop0
212+
# create a new LVM volume group on second disk
213+
pvcreate /dev/vdb
214+
vgcreate cinder-volumes /dev/vdb
166215
'';
167216
};
168217
};
@@ -173,12 +222,13 @@ in
173222
# Update: still does not work -.-
174223
environment.systemPackages = [
175224
pkgs.qemu
225+
pkgs.tgt
176226
];
177227

178228
systemd.services.cinder-volume = {
179229
description = "OpenStack Cinder Volume";
180230
after = [
181-
"cinder-volume-group.service"
231+
"cinder-volume-group-setup.service"
182232
];
183233
path = with pkgs; [
184234
cinder_env

‎modules/testing/README.md‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
# SSH Access
2+
3+
## Storage VM
4+
5+
```bash
6+
ssh root@localhost -p 2022 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
7+
```
8+
9+
## Controller VM
10+
11+
```bash
12+
ssh root@localhost -p 1122 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
13+
```
14+
15+
# Cinder & tgtd
16+
17+
## Create volume
18+
19+
* login in to `controllerVM` or `storageVM`
20+
21+
```bash
22+
openstack volume create --size 4 TEST_VOL
23+
```
24+
25+
* Cinder will create a LVM logical volume in the volume group `cinder-volumes`.
26+
* At this time this volume will not be exposed as an iSCSI volume. This only happens when the volume is assigned to a VM.
27+
28+
## Assign volume to a VM
29+
30+
* create a new VM with: `openstack server create`
31+
* or look into system unit: `openstack-create-vm` on VM `controllerVM`
32+
33+
* attach volume to vm
34+
35+
```bash
36+
openstack server list
37+
openstack volume list
38+
openstack server add volume <INSTANCE_NAME_OR_ID> <VOLUME_NAME_OR_ID>
39+
```
40+
41+
* Verify tgtd status on `storageVM`: `tgt-admin --dump`
42+
* Cinder should generated a tgtd configuration file within: `/var/lib/cinder/volumes`
43+
* In the VM `computeVM` should appear a new block device `sda`. (`dmesg`)
44+
45+
```bash
46+
[root@storageVM:/var/lib/cinder/volumes]# l
47+
total 12K
48+
drwxr-xr-x 2 cinder cinder 4.0K Feb 24 08:16 .
49+
drwxr-xr-x 6 cinder cinder 4.0K Feb 24 08:09 ..
50+
-rw------- 1 cinder cinder 268 Feb 24 08:16 volume-64159e0c-18bb-449f-b1e0-86f198b170e4
51+
```

‎modules/testing/default.nix‎

Lines changed: 84 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,26 @@ let
3333

3434
environment.systemPackages = [
3535
pkgs.openstackclient
36+
pkgs.openiscsi
37+
pkgs.sshpass
3638
];
3739

3840
environment.variables = adminEnv;
41+
42+
# enable easy access to test VMs with ssh
43+
users.users.root.openssh.authorizedKeys.keys = [
44+
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEMjGRp1/vaTxGiGbZnaSv4wu4LUUP7lGSGDFKfF31xw paul.kroeher@cyberus-technology.de"
45+
];
46+
47+
services.openssh = {
48+
enable = true;
49+
ports = [ 22 ];
50+
settings = {
51+
PasswordAuthentication = true;
52+
PermitRootLogin = "without-password"; # "yes", "without-password", "prohibit-password", "forced-commands-only", "no"
53+
};
54+
};
55+
3956
};
4057
};
4158

@@ -115,6 +132,10 @@ in
115132
vlan = 2;
116133
};
117134
};
135+
# enable ssh access
136+
forwardPorts = [
137+
{ from = "host"; host.port = 1122; guest.port = 22; }
138+
];
118139
};
119140

120141
systemd.services.openstack-create-vm = {
@@ -170,14 +191,13 @@ in
170191
matchConfig.Name = [ "eth0" ];
171192
networkConfig = {
172193
DHCP = "yes";
194+
DNS = "8.8.8.8";
173195
};
174196
};
175197
eth1 = {
176198
matchConfig.Name = [ "eth1" ];
177199
networkConfig = {
178200
Address = "10.0.0.11/24";
179-
Gateway = "10.0.0.1";
180-
DNS = "8.8.8.8";
181201
};
182202
};
183203

@@ -237,4 +257,66 @@ in
237257
};
238258

239259
};
260+
261+
testStorage =
262+
{ ... }:
263+
{
264+
265+
imports = [ common ];
266+
267+
virtualisation = {
268+
memorySize = 4096;
269+
cores = 4;
270+
diskSize = 4096;
271+
# add separate disk as LVM backend
272+
emptyDiskImages = [
273+
16384 # 16GB
274+
];
275+
interfaces = {
276+
eth1 = {
277+
vlan = 1;
278+
};
279+
eth2 = {
280+
vlan = 2;
281+
};
282+
};
283+
# enable ssh access
284+
forwardPorts = [
285+
{ from = "host"; host.port = 2022; guest.port = 22; }
286+
];
287+
};
288+
289+
systemd.network = {
290+
enable = true;
291+
wait-online.enable = false;
292+
293+
networks = {
294+
eth0 = {
295+
matchConfig.Name = [ "eth0" ];
296+
networkConfig = {
297+
DHCP = "yes";
298+
LinkLocalAddressing = "yes";
299+
KeepConfiguration = "yes";
300+
DNS = "8.8.8.8";
301+
};
302+
};
303+
304+
eth1 = {
305+
matchConfig.Name = [ "eth1" ];
306+
networkConfig = {
307+
Address = "10.0.0.20/24";
308+
};
309+
};
310+
311+
eth2 = {
312+
matchConfig.Name = [ "eth2" ];
313+
networkConfig = {
314+
DHCP = "no";
315+
LinkLocalAddressing = "no";
316+
KeepConfiguration = "yes";
317+
};
318+
};
319+
};
320+
};
321+
};
240322
}

0 commit comments

Comments
 (0)