Linstor fix migration while node offline (apache#8610)

* linstor: Add util method getBestErrorMessage from main

* linstor: failed remove of allow-two-primaries is no fatal error

* linstor: Fix failure if a Linstor node is down while migrating

If a Linstor node is down while migrating resource, allow-two-primaries
setting will fail because we can't reach the downed node. But it will
still set the property on the other nodes and migration should work.
We now just report an error instead of completely failing.

Author: rp-

plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java

@@ -276,27 +276,35 @@ public boolean connectPhysicalDisk(String volumePath, KVMStoragePool pool, Map<S
         }
 
         final DevelopersApi api = getLinstorAPI(pool);
+        String rscName;
         try
         {
-            final String rscName = getLinstorRscName(volumePath);
+            rscName = getLinstorRscName(volumePath);
 
             ResourceMakeAvailable rma = new ResourceMakeAvailable();
             ApiCallRcList answers = api.resourceMakeAvailableOnNode(rscName, localNodeName, rma);
             checkLinstorAnswersThrow(answers);
 
+        } catch (ApiException apiEx) {
+            logger.error(apiEx);
+            throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx);
+        }
+
+        try
+        {
             // allow 2 primaries for live migration, should be removed by disconnect on the other end
             ResourceDefinitionModify rdm = new ResourceDefinitionModify();
             Properties props = new Properties();
             props.put("DrbdOptions/Net/allow-two-primaries", "yes");
             rdm.setOverrideProps(props);
-            answers = api.resourceDefinitionModify(rscName, rdm);
+            ApiCallRcList answers = api.resourceDefinitionModify(rscName, rdm);
             if (answers.hasError()) {
                 logger.error("Unable to set 'allow-two-primaries' on " + rscName);
-                throw new CloudRuntimeException(answers.get(0).getMessage());
+                // do not fail here as adding allow-two-primaries property is only a problem while live migrating
             }
         } catch (ApiException apiEx) {
             logger.error(apiEx);
-            throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx);
+            // do not fail here as adding allow-two-primaries property is only a problem while live migrating
         }
         return true;
     }
@@ -360,19 +368,21 @@ public boolean disconnectPhysicalDiskByPath(String localPath)
                     ApiCallRcList answers = api.resourceDefinitionModify(rsc.get().getName(), rdm);
                     if (answers.hasError())
                     {
-                        logger.error("Failed to remove 'allow-two-primaries' on " + rsc.get().getName());
-                        throw new CloudRuntimeException(answers.get(0).getMessage());
+                        logger.error(
+                                String.format("Failed to remove 'allow-two-primaries' on %s: %s",
+                                        rsc.get().getName(), LinstorUtil.getBestErrorMessage(answers)));
+                        // do not fail here as removing allow-two-primaries property isn't fatal
                     }
 
                     return true;
                 }
                 logger.warn("Linstor: Couldn't find resource for this path: " + localPath);
             } catch (ApiException apiEx) {
-                logger.error(apiEx);
-                throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx);
+                logger.error(apiEx.getBestMessage());
+                // do not fail here as removing allow-two-primaries property isn't fatal
             }
         }
-        return false;
+        return true;
     }
 
     @Override

plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java

@@ -137,28 +137,33 @@ public static String getSnapshotPath(com.linbit.linstor.api.model.StoragePool sp
         return path;
     }
 
-    public static long getCapacityBytes(String linstorUrl, String rscGroupName) {
-        DevelopersApi linstorApi = getLinstorAPI(linstorUrl);
-        try {
-            List<ResourceGroup> rscGrps = linstorApi.resourceGroupList(
+    public static List<StoragePool> getRscGroupStoragePools(DevelopersApi api, String rscGroupName)
+            throws ApiException {
+        List<ResourceGroup> rscGrps = api.resourceGroupList(
                 Collections.singletonList(rscGroupName),
                 null,
                 null,
                 null);
 
-            if (rscGrps.isEmpty()) {
-                final String errMsg = String.format("Linstor: Resource group '%s' not found", rscGroupName);
-                LOGGER.error(errMsg);
-                throw new CloudRuntimeException(errMsg);
-            }
+        if (rscGrps.isEmpty()) {
+            final String errMsg = String.format("Linstor: Resource group '%s' not found", rscGroupName);
+            LOGGER.error(errMsg);
+            throw new CloudRuntimeException(errMsg);
+        }
 
-            List<StoragePool> storagePools = linstorApi.viewStoragePools(
+        return api.viewStoragePools(
                 Collections.emptyList(),
                 rscGrps.get(0).getSelectFilter().getStoragePoolList(),
                 null,
                 null,
                 null
-            );
+        );
+    }
+
+    public static long getCapacityBytes(String linstorUrl, String rscGroupName) {
+        DevelopersApi linstorApi = getLinstorAPI(linstorUrl);
+        try {
+            List<StoragePool> storagePools = getRscGroupStoragePools(linstorApi, rscGroupName);
 
             return storagePools.stream()
                 .filter(sp -> sp.getProviderKind() != ProviderKind.DISKLESS)
@@ -169,4 +174,4 @@ public static long getCapacityBytes(String linstorUrl, String rscGroupName) {
             throw new CloudRuntimeException(apiEx);
         }
     }
-}
+}