FR79: Introducing storage tags on host, cluster and pods (apache#503)

The feature introduces Storage Access Groups on zone/cluster scoped storage pools and hosts/clusters/pods/zones to define the host and storage pool connections. Previously when a zone scoped storage pool is added it would connect to all the hosts in that zone, now with this feature one can define the scope of the connections of the storage to the required hosts.

* API and DB changes

* Added validations on vm and volume migrations wrt storage tags on host

* Fix tags during startupcommand

* update storagepool tags

* Added responses and UI changes

* Fix storage tags on cluster and pods

* some more fixes

* Fix storage tag deletion and tests

* Fix storage tags deletion

* Add & Remove PowerFlex/ScaleIO MDMs while preparing & unpreparing the storage SDC connections (instead of start & stop scini)

* storage pool detail null check

* Fix some tag edit scenarios

* Addressed review comments

* UI change for storage tags

* Fix empty storage tags on hosts validation during vm migrate

* Fix updating storage connection when applying tags on storage pool

* Fix to populate tags to hosts when tags are removed on cluster/pod

* Fix mysql queries

* Fixes for host tags and storage pool tags

* Add check for used volumes while updating storage pool

* Fix update cluster UI for tags deletion

* Fix update pod form to accept empty storage tags

* Zone storage pool tags introduction

* Added zone level storage tags

* Add storage tags to zone UI details

* Replace the usage of tags in storage pool with a new parameter for host connection

* Update host and storage connections based on consideration on empty tags

* Add hostId to liststoragepools API to list connected pools

* Few fixes and improvements

* Rename table

* Fix unit tests

* fix gen toc

* Add configureStorageAccess API to gen doc

* Change name from storagetags to storageaccessgroups

* Remove storagetags from all update APIs, just to use configureStorageAccess API

* Rename storage tags with storage access groups

* UI changes

* Add primary storage update UI

* UI changes

* Add proper migration checks and fix tests

* Add missing UI label

* Reorder mysql changes

* storage pool detail null check

* Add/Remove MDM IP addresses during Host connection/disconnection to/from storage pool when powerflex.connect.on.demand is false

* unit test fixes

* Fix storage access groups comparision

* Add check if all hosts are using the tags

* Fix error message

* Use listeners disconnect method

* Don't remove MDM IPs from SDC when any volumes mapped to SDC

* Fix hostDao

* Fix storage pool addition with the combined tags

* Addressed Review comments mainly on renaming

* Fix tests and some logic

* optimise the storage access group check

* Add tests for storage access group checks

* Added tests for volume migration checks

* Unit tests for checking all hosts using storage access groups

* Add tests for volumes in use by pools which needs to be disconnected

* Fix podId

* Add inherited storage groups to response and UI

* Fetch IDs from host object

* changes after rebasing apple-base418

* Don't remove MDM IPs when other pools of same ScaleIO/PowerFlex cluster are connected

* Add logs and a unit test

* Action event for configureStorageAccess API

* fetch eligible hosts while finding the endpoint for command executions

* Update storage access groups on host after Disconnected and Downstate

* Check for common accessible host for migrations

* update changes, to not remove/disconnect MDMs on maintenance

* Add volumes information in logging for better understanding

* Fix UI double notification

---------

Co-authored-by: Suresh Kumar Anaparti <suresh.anaparti@shapeblue.com>
Co-authored-by: Suresh Kumar Anaparti <sureshkumar.anaparti@gmail.com>

Author: harikrishna-patnala

api/src/main/java/com/cloud/configuration/ConfigurationService.java

@@ -187,25 +187,19 @@ public interface ConfigurationService {
     /**
      * Creates a new pod based on the parameters specified in the command object
      *
-     * @param zoneId
-     *            TODO
-     * @param name
-     *            TODO
-     * @param startIp
-     *            TODO
-     * @param endIp
-     *            TODO
-     * @param gateway
-     *            TODO
-     * @param netmask
-     *            TODO
-     * @param allocationState
-     *            TODO
+     * @param zoneId              TODO
+     * @param name                TODO
+     * @param startIp             TODO
+     * @param endIp               TODO
+     * @param gateway             TODO
+     * @param netmask             TODO
+     * @param allocationState     TODO
+     * @param storageAccessGroups
      * @return the new pod if successful, null otherwise
      * @throws
      * @throws
      */
-    Pod createPod(long zoneId, String name, String startIp, String endIp, String gateway, String netmask, String allocationState);
+    Pod createPod(long zoneId, String name, String startIp, String endIp, String gateway, String netmask, String allocationState, List<String> storageAccessGroups);
 
     /**
      * Creates a mutual exclusive IP range in the pod with same gateway, netmask.

api/src/main/java/com/cloud/dc/Pod.java

@@ -43,4 +43,6 @@ public interface Pod extends InfrastructureEntity, Grouping, Identity, InternalI
     AllocationState getAllocationState();
 
     boolean getExternalDhcp();
+
+    String getStorageAccessGroups();
 }

api/src/main/java/com/cloud/event/EventTypes.java

@@ -430,6 +430,8 @@ public class EventTypes {
     public static final String EVENT_DISABLE_PRIMARY_STORAGE = "DISABLE.PS";
     public static final String EVENT_SYNC_STORAGE_POOL = "SYNC.STORAGE.POOL";
 
+    public static final String EVENT_CONFIGURE_STORAGE_ACCESS = "CONFIGURE.STORAGE.ACCESS";
+
     // VPN
     public static final String EVENT_REMOTE_ACCESS_VPN_CREATE = "VPN.REMOTE.ACCESS.CREATE";
     public static final String EVENT_REMOTE_ACCESS_VPN_DESTROY = "VPN.REMOTE.ACCESS.DESTROY";

api/src/main/java/com/cloud/host/Host.java

@@ -209,4 +209,6 @@ public static String[] toStrings(Host.Type... types) {
     boolean isDisabled();
 
     ResourceState getResourceState();
+
+    String getStorageAccessGroups();
 }

api/src/main/java/com/cloud/org/Cluster.java

@@ -38,4 +38,6 @@ public static enum ClusterType {
     AllocationState getAllocationState();
 
     ManagedState getManagedState();
+
+    String getStorageAccessGroups();
 }

api/src/main/java/com/cloud/resource/ResourceService.java

@@ -95,4 +95,11 @@ public interface ResourceService {
 
     boolean releaseHostReservation(Long hostId);
 
+    void updatePodStorageAccessGroups(long podId, List<String> newStorageAccessGroups);
+
+    void updateZoneStorageAccessGroups(long zoneId, List<String> newStorageAccessGroups);
+
+    void updateClusterStorageAccessGroups(Long clusterId, List<String> newStorageAccessGroups);
+
+    void updateHostStorageAccessGroups(Long hostId, List<String> newStorageAccessGroups);
 }

api/src/main/java/com/cloud/storage/StorageService.java

@@ -21,6 +21,7 @@
 import java.util.Map;
 
 import org.apache.cloudstack.api.command.admin.storage.CancelPrimaryStorageMaintenanceCmd;
+import org.apache.cloudstack.api.command.admin.storage.ConfigureStorageAccessCmd;
 import org.apache.cloudstack.api.command.admin.storage.CreateSecondaryStagingStoreCmd;
 import org.apache.cloudstack.api.command.admin.storage.CreateStoragePoolCmd;
 import org.apache.cloudstack.api.command.admin.storage.DeleteImageStoreCmd;
@@ -85,6 +86,8 @@ public interface StorageService {
 
     StoragePool updateStoragePool(UpdateStoragePoolCmd cmd) throws IllegalArgumentException;
 
+    boolean configureStorageAccess(ConfigureStorageAccessCmd cmd);
+
     StoragePool getStoragePool(long id);
 
     boolean deleteImageStore(DeleteImageStoreCmd cmd);

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -434,6 +434,7 @@ public class ApiConstants {
     public static final String SYSTEM_VM_TYPE = "systemvmtype";
     public static final String TAGS = "tags";
     public static final String STORAGE_TAGS = "storagetags";
+    public static final String STORAGE_ACCESS_GROUPS = "storageaccessgroups";
     public static final String TARGET_IQN = "targetiqn";
     public static final String TEMPLATE_FILTER = "templatefilter";
     public static final String TEMPLATE_ID = "templateid";

api/src/main/java/org/apache/cloudstack/api/command/admin/cluster/AddClusterCmd.java

@@ -114,6 +114,12 @@ public class AddClusterCmd extends BaseCmd {
     private String ovm3cluster;
     @Parameter(name = ApiConstants.OVM3_VIP, type = CommandType.STRING, required = false,  description = "Ovm3 vip to use for pool (and cluster)")
     private String ovm3vip;
+    @Parameter(name = ApiConstants.STORAGE_ACCESS_GROUPS,
+            type = CommandType.LIST, collectionType = CommandType.STRING,
+            description = "comma separated list of storage access groups for the hosts in the cluster",
+            since = "4.18.1")
+    private List<String> storageAccessGroups;
+
     public String getOvm3Pool() {
          return ovm3pool;
     }
@@ -188,6 +194,10 @@ public void setClusterType(String type) {
         this.clusterType = type;
     }
 
+    public List<String> getStorageAccessGroups() {
+        return storageAccessGroups;
+    }
+
     @Override
     public long getEntityOwnerId() {
         return Account.ACCOUNT_ID_SYSTEM;

api/src/main/java/org/apache/cloudstack/api/command/admin/cluster/UpdateClusterCmd.java

@@ -118,7 +118,7 @@ public void execute() {
         }
         Cluster result = _resourceService.updateCluster(this);
         if (result != null) {
-            ClusterResponse clusterResponse = _responseGenerator.createClusterResponse(cluster, false);
+            ClusterResponse clusterResponse = _responseGenerator.createClusterResponse(result, false);
             clusterResponse.setResponseName(getCommandName());
             this.setResponseObject(clusterResponse);
         } else {