Skip to content

Commit ff961c9

Browse files
rp-rp-
authored
linstor: support QoS(IOPs) and small improvements (#6682)
This PR has 3 improvements for the Linstor primary storage driver: - Create a separate jar of it and move all Linstor related classes into the correct project (similar to the storpool plugin) - Add aux properties for Cloudstack volumes in Linstor to make it easier to identify them in Linstor - Add support for IOPs settings with the Linstor storage plugin
1 parent b8d834e commit ff961c9

File tree

11 files changed

+142
-20
lines changed

11 files changed

+142
-20
lines changed

‎client/pom.xml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -767,6 +767,12 @@
767767
<overWrite>false</overWrite>
768768
<outputDirectory>${project.build.directory}/lib</outputDirectory>
769769
</artifactItem>
770+
<artifactItem>
771+
<groupId>org.apache.cloudstack</groupId>
772+
<artifactId>cloud-plugin-storage-volume-linstor</artifactId>
773+
<overWrite>false</overWrite>
774+
<outputDirectory>${project.build.directory}/lib</outputDirectory>
775+
</artifactItem>
770776
<artifactItem>
771777
<groupId>org.bouncycastle</groupId>
772778
<artifactId>bctls-jdk15on</artifactId>
@@ -811,6 +817,7 @@
811817
<exclude>org.bouncycastle:bctls-jdk15on</exclude>
812818
<exclude>mysql:mysql-connector-java</exclude>
813819
<exclude>org.apache.cloudstack:cloud-plugin-storage-volume-storpool</exclude>
820+
<exclude>org.apache.cloudstack:cloud-plugin-storage-volume-linstor</exclude>
814821
</excludes>
815822
</artifactSet>
816823
<transformers>

‎debian/rules‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ override_dh_auto_install:
4242
install -D plugins/hypervisors/kvm/target/cloud-plugin-hypervisor-kvm-$(VERSION).jar $(DESTDIR)/usr/share/$(PACKAGE)-agent/lib/
4343
install -D plugins/hypervisors/kvm/target/dependencies/* $(DESTDIR)/usr/share/$(PACKAGE)-agent/lib/
4444
install -D plugins/storage/volume/storpool/target/cloud-plugin-storage-volume-storpool-$(VERSION).jar $(DESTDIR)/usr/share/$(PACKAGE)-agent/lib/
45+
install -D plugins/storage/volume/linstor/target/cloud-plugin-storage-volume-linstor-$(VERSION).jar $(DESTDIR)/usr/share/$(PACKAGE)-agent/lib/
4546

4647
install -d -m0755 debian/$(PACKAGE)-agent/lib/systemd/system
4748
install -m0644 packaging/systemd/$(PACKAGE)-agent.service debian/$(PACKAGE)-agent/lib/systemd/system/$(PACKAGE)-agent.service

‎packaging/centos7/cloud.spec‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -355,6 +355,7 @@ install -D agent/target/transformed/cloudstack-agent.logrotate ${RPM_BUILD_ROOT}
355355
install -D plugins/hypervisors/kvm/target/cloud-plugin-hypervisor-kvm-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%name-agent/lib/cloud-plugin-hypervisor-kvm-%{_maventag}.jar
356356
cp plugins/hypervisors/kvm/target/dependencies/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib
357357
cp plugins/storage/volume/storpool/target/*.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib
358+
cp plugins/storage/volume/linstor/target/*.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib
358359

359360
# Usage server
360361
mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/usage

‎packaging/centos8/cloud.spec‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -348,6 +348,7 @@ install -D agent/target/transformed/cloudstack-agent.logrotate ${RPM_BUILD_ROOT}
348348
install -D plugins/hypervisors/kvm/target/cloud-plugin-hypervisor-kvm-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%name-agent/lib/cloud-plugin-hypervisor-kvm-%{_maventag}.jar
349349
cp plugins/hypervisors/kvm/target/dependencies/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib
350350
cp plugins/storage/volume/storpool/target/*.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib
351+
cp plugins/storage/volume/linstor/target/*.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib
351352

352353
# Usage server
353354
mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/usage

‎packaging/suse15/cloud.spec‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -350,6 +350,7 @@ install -D agent/target/transformed/cloudstack-agent.logrotate ${RPM_BUILD_ROOT}
350350
install -D plugins/hypervisors/kvm/target/cloud-plugin-hypervisor-kvm-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%name-agent/lib/cloud-plugin-hypervisor-kvm-%{_maventag}.jar
351351
cp plugins/hypervisors/kvm/target/dependencies/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib
352352
cp plugins/storage/volume/storpool/target/*.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib
353+
cp plugins/storage/volume/linstor/target/*.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib
353354

354355
# Usage server
355356
mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/usage

‎plugins/storage/volume/linstor/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,11 @@
3838
<artifactId>java-linstor</artifactId>
3939
<version>${cs.java-linstor.version}</version>
4040
</dependency>
41+
<dependency>
42+
<groupId>org.apache.cloudstack</groupId>
43+
<artifactId>cloud-plugin-hypervisor-kvm</artifactId>
44+
<version>${project.version}</version>
45+
</dependency>
4146
</dependencies>
4247
<build>
4348
<plugins>

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java renamed to plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ private String getLinstorRscName(String name) {
7171

7272
private String getHostname() {
7373
// either there is already some function for that in the agent or a better way.
74-
ProcessBuilder pb = new ProcessBuilder("hostname");
74+
ProcessBuilder pb = new ProcessBuilder("/usr/bin/hostname");
7575
try
7676
{
7777
String result;
@@ -86,7 +86,8 @@ private String getHostname() {
8686
p.destroy();
8787
return result.trim();
8888
} catch (IOException | InterruptedException exc) {
89-
throw new CloudRuntimeException("Unable to run 'hostname' command.");
89+
Thread.currentThread().interrupt();
90+
throw new CloudRuntimeException("Unable to run '/usr/bin/hostname' command.");
9091
}
9192
}
9293

@@ -310,11 +311,11 @@ private Optional<ResourceWithVolumes> getResourceByPath(final List<ResourceWithV
310311
public boolean disconnectPhysicalDiskByPath(String localPath)
311312
{
312313
// get first storage pool from the map, as we don't know any better:
313-
if (!MapStorageUuidToStoragePool.isEmpty())
314+
Optional<KVMStoragePool> optFirstPool = MapStorageUuidToStoragePool.values().stream().findFirst();
315+
if (optFirstPool.isPresent())
314316
{
315317
s_logger.debug("Linstor: disconnectPhysicalDiskByPath " + localPath);
316-
String firstKey = MapStorageUuidToStoragePool.keySet().stream().findFirst().get();
317-
final KVMStoragePool pool = MapStorageUuidToStoragePool.get(firstKey);
318+
final KVMStoragePool pool = optFirstPool.get();
318319

319320
s_logger.debug("Linstor: Using storpool: " + pool.getUuid());
320321
final DevelopersApi api = getLinstorAPI(pool);

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStoragePool.java renamed to plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStoragePool.java

File renamed without changes.

‎plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java‎

Lines changed: 105 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -21,22 +21,28 @@
2121
import com.linbit.linstor.api.DevelopersApi;
2222
import com.linbit.linstor.api.model.ApiCallRc;
2323
import com.linbit.linstor.api.model.ApiCallRcList;
24+
import com.linbit.linstor.api.model.Properties;
2425
import com.linbit.linstor.api.model.ResourceDefinition;
2526
import com.linbit.linstor.api.model.ResourceDefinitionCloneRequest;
2627
import com.linbit.linstor.api.model.ResourceDefinitionCloneStarted;
2728
import com.linbit.linstor.api.model.ResourceDefinitionCreate;
29+
import com.linbit.linstor.api.model.ResourceDefinitionModify;
2830
import com.linbit.linstor.api.model.ResourceGroupSpawn;
2931
import com.linbit.linstor.api.model.ResourceWithVolumes;
3032
import com.linbit.linstor.api.model.Snapshot;
3133
import com.linbit.linstor.api.model.SnapshotRestore;
34+
import com.linbit.linstor.api.model.VolumeDefinition;
3235
import com.linbit.linstor.api.model.VolumeDefinitionModify;
3336

3437
import javax.annotation.Nonnull;
3538
import javax.inject.Inject;
39+
40+
import java.util.Arrays;
3641
import java.util.Collections;
3742
import java.util.HashMap;
3843
import java.util.List;
3944
import java.util.Map;
45+
import java.util.Objects;
4046

4147
import com.cloud.agent.api.Answer;
4248
import com.cloud.agent.api.storage.ResizeVolumeAnswer;
@@ -242,14 +248,17 @@ public void deleteAsync(DataStore dataStore, DataObject dataObject, AsyncComplet
242248
deleteResourceDefinition(storagePool, rscName);
243249

244250
long usedBytes = storagePool.getUsedBytes();
245-
long capacityIops = storagePool.getCapacityIops();
251+
Long capacityIops = storagePool.getCapacityIops();
246252

247-
usedBytes -= volumeInfo.getSize();
248-
if (volumeInfo.getMaxIops() != null)
249-
capacityIops += volumeInfo.getMaxIops();
253+
if (capacityIops != null)
254+
{
255+
if (volumeInfo.getMaxIops() != null)
256+
capacityIops += volumeInfo.getMaxIops();
257+
storagePool.setCapacityIops(Math.max(0, capacityIops));
258+
}
250259

260+
usedBytes -= volumeInfo.getSize();
251261
storagePool.setUsedBytes(Math.max(0, usedBytes));
252-
storagePool.setCapacityIops(Math.max(0, capacityIops));
253262

254263
_storagePoolDao.update(storagePoolId, storagePool);
255264
}
@@ -325,6 +334,72 @@ private String getDeviceName(DevelopersApi linstorApi, String rscName) throws Ap
325334
}
326335
}
327336

337+
private void applyQoSSettings(StoragePoolVO storagePool, DevelopersApi api, String rscName, Long maxIops)
338+
throws ApiException
339+
{
340+
Long currentQosIops = null;
341+
List<VolumeDefinition> vlmDfns = api.volumeDefinitionList(rscName, null, null);
342+
if (!vlmDfns.isEmpty())
343+
{
344+
Properties props = vlmDfns.get(0).getProps();
345+
long iops = Long.parseLong(props.getOrDefault("sys/fs/blkio_throttle_write_iops", "0"));
346+
currentQosIops = iops > 0 ? iops : null;
347+
}
348+
349+
if (!Objects.equals(maxIops, currentQosIops))
350+
{
351+
VolumeDefinitionModify vdm = new VolumeDefinitionModify();
352+
if (maxIops != null)
353+
{
354+
Properties props = new Properties();
355+
props.put("sys/fs/blkio_throttle_read_iops", "" + maxIops);
356+
props.put("sys/fs/blkio_throttle_write_iops", "" + maxIops);
357+
vdm.overrideProps(props);
358+
s_logger.info("Apply qos setting: " + maxIops + " to " + rscName);
359+
}
360+
else
361+
{
362+
s_logger.info("Remove QoS setting for " + rscName);
363+
vdm.deleteProps(Arrays.asList("sys/fs/blkio_throttle_read_iops", "sys/fs/blkio_throttle_write_iops"));
364+
}
365+
ApiCallRcList answers = api.volumeDefinitionModify(rscName, 0, vdm);
366+
checkLinstorAnswersThrow(answers);
367+
368+
Long capacityIops = storagePool.getCapacityIops();
369+
if (capacityIops != null)
370+
{
371+
long vcIops = currentQosIops != null ? currentQosIops * -1 : 0;
372+
long vMaxIops = maxIops != null ? maxIops : 0;
373+
long newIops = vcIops + vMaxIops;
374+
capacityIops -= newIops;
375+
s_logger.info("Current storagepool " + storagePool.getName() + " iops capacity: " + capacityIops);
376+
storagePool.setCapacityIops(Math.max(0, capacityIops));
377+
_storagePoolDao.update(storagePool.getId(), storagePool);
378+
}
379+
}
380+
}
381+
382+
private void applyAuxProps(DevelopersApi api, String rscName, String dispName, String vmName)
383+
throws ApiException
384+
{
385+
ResourceDefinitionModify rdm = new ResourceDefinitionModify();
386+
Properties props = new Properties();
387+
if (dispName != null)
388+
{
389+
props.put("Aux/cs-name", dispName);
390+
}
391+
if (vmName != null)
392+
{
393+
props.put("Aux/cs-vm-name", vmName);
394+
}
395+
if (!props.isEmpty())
396+
{
397+
rdm.setOverrideProps(props);
398+
ApiCallRcList answers = api.resourceDefinitionModify(rscName, rdm);
399+
checkLinstorAnswersThrow(answers);
400+
}
401+
}
402+
328403
private String createResource(VolumeInfo vol, StoragePoolVO storagePoolVO)
329404
{
330405
DevelopersApi linstorApi = LinstorUtil.getLinstorAPI(storagePoolVO.getHostAddress());
@@ -338,10 +413,13 @@ private String createResource(VolumeInfo vol, StoragePoolVO storagePoolVO)
338413

339414
try
340415
{
341-
s_logger.debug("Linstor: Spawn resource " + rscName);
416+
s_logger.info("Linstor: Spawn resource " + rscName);
342417
ApiCallRcList answers = linstorApi.resourceGroupSpawn(rscGrp, rscGrpSpawn);
343418
checkLinstorAnswersThrow(answers);
344419

420+
applyAuxProps(linstorApi, rscName, vol.getName(), vol.getAttachedVmName());
421+
applyQoSSettings(storagePoolVO, linstorApi, rscName, vol.getMaxIops());
422+
345423
return getDeviceName(linstorApi, rscName);
346424
} catch (ApiException apiEx)
347425
{
@@ -361,7 +439,7 @@ private String cloneResource(long csCloneId, VolumeInfo volumeInfo, StoragePoolV
361439
final DevelopersApi linstorApi = LinstorUtil.getLinstorAPI(storagePoolVO.getHostAddress());
362440

363441
try {
364-
s_logger.debug("Clone resource definition " + cloneRes + " to " + rscName);
442+
s_logger.info("Clone resource definition " + cloneRes + " to " + rscName);
365443
ResourceDefinitionCloneRequest cloneRequest = new ResourceDefinitionCloneRequest();
366444
cloneRequest.setName(rscName);
367445
ResourceDefinitionCloneStarted cloneStarted = linstorApi.resourceDefinitionClone(
@@ -373,6 +451,10 @@ private String cloneResource(long csCloneId, VolumeInfo volumeInfo, StoragePoolV
373451
throw new CloudRuntimeException("Clone for resource " + rscName + " failed.");
374452
}
375453

454+
s_logger.info("Clone resource definition " + cloneRes + " to " + rscName + " finished");
455+
applyAuxProps(linstorApi, rscName, volumeInfo.getName(), volumeInfo.getAttachedVmName());
456+
applyQoSSettings(storagePoolVO, linstorApi, rscName, volumeInfo.getMaxIops());
457+
376458
return getDeviceName(linstorApi, rscName);
377459
} catch (ApiException apiEx) {
378460
s_logger.error("Linstor: ApiEx - " + apiEx.getMessage());
@@ -413,10 +495,13 @@ private String createResourceFromSnapshot(long csSnapshotId, String rscName, Sto
413495
checkLinstorAnswersThrow(answers);
414496

415497
// restore snapshot to new resource
416-
s_logger.debug("Restore resource from snapshot: " + cloneRes + ":" + snapName);
498+
s_logger.info("Restore resource from snapshot: " + cloneRes + ":" + snapName);
417499
answers = linstorApi.resourceSnapshotRestore(cloneRes, snapName, snapshotRestore);
418500
checkLinstorAnswersThrow(answers);
419501

502+
applyAuxProps(linstorApi, rscName, volumeVO.getName(), null);
503+
applyQoSSettings(storagePoolVO, linstorApi, rscName, volumeVO.getMaxIops());
504+
420505
return getDeviceName(linstorApi, rscName);
421506
} catch (ApiException apiEx) {
422507
s_logger.error("Linstor: ApiEx - " + apiEx.getMessage());
@@ -608,12 +693,11 @@ public void copyAsync(DataObject srcData, DataObject destData, Host destHost, As
608693
}
609694

610695
private CreateCmdResult notifyResize(
611-
DataObject data,
696+
VolumeObject vol,
612697
long oldSize,
613698
ResizeVolumePayload resizeParameter)
614699
{
615-
VolumeObject vol = (VolumeObject) data;
616-
StoragePool pool = (StoragePool) data.getDataStore();
700+
StoragePool pool = (StoragePool) vol.getDataStore();
617701

618702
ResizeVolumeCommand resizeCmd =
619703
new ResizeVolumeCommand(vol.getPath(), new StorageFilerTO(pool), oldSize, resizeParameter.newSize, resizeParameter.shrinkOk,
@@ -642,7 +726,7 @@ private CreateCmdResult notifyResize(
642726
public void resize(DataObject data, AsyncCompletionCallback<CreateCmdResult> callback)
643727
{
644728
final VolumeObject vol = (VolumeObject) data;
645-
final StoragePool pool = (StoragePool) data.getDataStore();
729+
final StoragePoolVO pool = _storagePoolDao.findById(data.getDataStore().getId());
646730
final DevelopersApi api = LinstorUtil.getLinstorAPI(pool.getHostAddress());
647731
final ResizeVolumePayload resizeParameter = (ResizeVolumePayload) vol.getpayload();
648732

@@ -654,6 +738,14 @@ public void resize(DataObject data, AsyncCompletionCallback<CreateCmdResult> cal
654738
dfm.setSizeKib(resizeParameter.newSize / 1024);
655739
try
656740
{
741+
applyQoSSettings(pool, api, rscName, resizeParameter.newMaxIops);
742+
{
743+
final VolumeVO volume = _volumeDao.findById(vol.getId());
744+
volume.setMinIops(resizeParameter.newMinIops);
745+
volume.setMaxIops(resizeParameter.newMaxIops);
746+
_volumeDao.update(volume.getId(), volume);
747+
}
748+
657749
ApiCallRcList answers = api.volumeDefinitionModify(rscName, 0, dfm);
658750
if (answers.hasError())
659751
{
@@ -680,7 +772,7 @@ public void resize(DataObject data, AsyncCompletionCallback<CreateCmdResult> cal
680772
} else
681773
{
682774
// notify guests
683-
result = notifyResize(data, oldSize, resizeParameter);
775+
result = notifyResize(vol, oldSize, resizeParameter);
684776
}
685777

686778
callback.complete(result);

‎plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/LinstorPrimaryDataStoreLifeCycleImpl.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ public DataStore initialize(Map<String, Object> dsInfos) {
8989
Long clusterId = (Long) dsInfos.get("clusterId");
9090
String storagePoolName = (String) dsInfos.get("name");
9191
String providerName = (String) dsInfos.get("providerName");
92+
Long capacityIops = (Long) dsInfos.get("capacityIops");
9293
String tags = (String) dsInfos.get("tags");
9394
@SuppressWarnings("unchecked")
9495
Map<String, String> details = (Map<String, String>) dsInfos.get("details");
@@ -145,11 +146,14 @@ public DataStore initialize(Map<String, Object> dsInfos) {
145146
}
146147

147148
long capacityBytes = LinstorUtil.getCapacityBytes(url, resourceGroup);
148-
149149
if (capacityBytes <= 0) {
150150
throw new IllegalArgumentException("'capacityBytes' must be present and greater than 0.");
151151
}
152152

153+
if (capacityIops != null) {
154+
parameters.setCapacityIops(capacityIops);
155+
}
156+
153157
parameters.setHost(url);
154158
parameters.setPort(port);
155159
parameters.setPath(resourceGroup);
@@ -161,7 +165,7 @@ public DataStore initialize(Map<String, Object> dsInfos) {
161165
parameters.setManaged(false);
162166
parameters.setCapacityBytes(capacityBytes);
163167
parameters.setUsedBytes(0);
164-
parameters.setCapacityIops(0L);
168+
parameters.setCapacityIops(capacityIops);
165169
parameters.setHypervisorType(HypervisorType.KVM);
166170
parameters.setTags(tags);
167171
parameters.setDetails(details);

0 commit comments

Comments
 (0)