Merge remote-tracking branch 'apache/main' into 4.20-vr-python3

Author: weizhouapache

.github/workflows/ci.yml

@@ -179,7 +179,8 @@ jobs:
                 "component/test_project_usage
                   component/test_protocol_number_security_group
                   component/test_public_ip
-                  component/test_resource_limits",
+                  component/test_resource_limits
+                  component/test_resource_limit_tags",
                 "component/test_regions_accounts
                   component/test_routers
                   component/test_snapshots

api/src/main/java/com/cloud/capacity/Capacity.java

@@ -16,6 +16,8 @@
 // under the License.
 package com.cloud.capacity;
 
+import java.util.List;
+
 import org.apache.cloudstack.api.Identity;
 import org.apache.cloudstack.api.InternalIdentity;
 
@@ -35,6 +37,11 @@ public interface Capacity extends InternalIdentity, Identity {
 
     public static final short CAPACITY_TYPE_CPU_CORE = 90;
 
+    public static final List<Short> STORAGE_CAPACITY_TYPES = List.of(CAPACITY_TYPE_STORAGE,
+            CAPACITY_TYPE_STORAGE_ALLOCATED,
+            CAPACITY_TYPE_SECONDARY_STORAGE,
+            CAPACITY_TYPE_LOCAL_STORAGE);
+
     public Long getHostOrPoolId();
 
     public Long getDataCenterId();
@@ -54,4 +61,6 @@ public interface Capacity extends InternalIdentity, Identity {
     public Float getUsedPercentage();
 
     public Long getAllocatedCapacity();
+
+    public String getTag();
 }

api/src/main/java/com/cloud/configuration/Resource.java

@@ -85,5 +85,6 @@ public String getName() {
     long getOwnerId();
 
     ResourceOwnerType getResourceOwnerType();
+    String getTag();
 
 }

api/src/main/java/com/cloud/server/ManagementService.java

@@ -449,10 +449,11 @@ public interface ManagementService {
      * this method removes the child storage pools and adds the corresponding parent datastore cluster for API response listing
      *
      * @param Long volumeId
+     * @param String keyword if passed, will only return storage pools that contain this keyword in the name
      * @return Pair<List<? extends StoragePool>, List<? extends StoragePool>> List of storage pools in cluster and list
      *         of pools with enough capacity.
      */
-    Pair<List<? extends StoragePool>, List<? extends StoragePool>> listStoragePoolsForMigrationOfVolume(Long volumeId);
+    Pair<List<? extends StoragePool>, List<? extends StoragePool>> listStoragePoolsForMigrationOfVolume(Long volumeId, String keyword);
 
     Pair<List<? extends StoragePool>, List<? extends StoragePool>> listStoragePoolsForSystemMigrationOfVolume(Long volumeId, Long newDiskOfferingId, Long newSize, Long newMinIops, Long newMaxIops, boolean keepSourceStoragePool, boolean bypassStorageTypeCheck);
 

api/src/main/java/com/cloud/storage/Volume.java

@@ -30,6 +30,8 @@
 
 public interface Volume extends ControlledEntity, Identity, InternalIdentity, BasedOn, StateObject<Volume.State>, Displayable {
 
+    static final long DISK_OFFERING_SUITABILITY_CHECK_VOLUME_ID = -1;
+
     // Managed storage volume parameters (specified in the compute/disk offering for PowerFlex)
     String BANDWIDTH_LIMIT_IN_MBPS = "bandwidthLimitInMbps";
     String IOPS_LIMIT = "iopsLimit";

api/src/main/java/com/cloud/user/ResourceLimitService.java

@@ -18,13 +18,18 @@
 
 import java.util.List;
 
+import org.apache.cloudstack.api.response.AccountResponse;
+import org.apache.cloudstack.api.response.DomainResponse;
+import org.apache.cloudstack.framework.config.ConfigKey;
+
 import com.cloud.configuration.Resource.ResourceType;
 import com.cloud.configuration.ResourceCount;
 import com.cloud.configuration.ResourceLimit;
 import com.cloud.domain.Domain;
 import com.cloud.exception.ResourceAllocationException;
-import org.apache.cloudstack.framework.config.ConfigKey;
-import org.apache.cloudstack.user.ResourceReservation;
+import com.cloud.offering.DiskOffering;
+import com.cloud.offering.ServiceOffering;
+import com.cloud.template.VirtualMachineTemplate;
 
 public interface ResourceLimitService {
 
@@ -34,6 +39,13 @@ public interface ResourceLimitService {
             "The default maximum secondary storage space (in GiB) that can be used for a project", false);
     static final ConfigKey<Long> ResourceCountCheckInterval = new ConfigKey<>("Advanced", Long.class, "resourcecount.check.interval", "300",
             "Time (in seconds) to wait before running resource recalculation and fixing task. Default is 300 seconds, Setting this to 0 disables execution of the task", false);
+    static final ConfigKey<String> ResourceLimitHostTags = new ConfigKey<>("Advanced", String.class, "resource.limit.host.tags", "",
+            "A comma-separated list of tags for host resource limits", true);
+    static final ConfigKey<String> ResourceLimitStorageTags = new ConfigKey<>("Advanced", String.class, "resource.limit.storage.tags", "",
+            "A comma-separated list of tags for storage resource limits", true);
+
+    static final List<ResourceType> HostTagsSupportingTypes = List.of(ResourceType.user_vm, ResourceType.cpu, ResourceType.memory);
+    static final List<ResourceType> StorageTagsSupportingTypes = List.of(ResourceType.volume, ResourceType.primary_storage);
 
     /**
      * Updates an existing resource limit with the specified details. If a limit doesn't exist, will create one.
@@ -46,22 +58,27 @@ public interface ResourceLimitService {
      *            TODO
      * @param max
      *            TODO
+     * @param tag
+     *            tag for the resource type
      *
      * @return the updated/created resource limit
      */
-    ResourceLimit updateResourceLimit(Long accountId, Long domainId, Integer resourceType, Long max);
+    ResourceLimit updateResourceLimit(Long accountId, Long domainId, Integer resourceType, Long max, String tag);
 
     /**
      * Updates an existing resource count details for the account/domain
      *
      * @param accountId
-     *            TODO
+     *            Id of the account for which resource recalculation to be done
      * @param domainId
-     *            TODO
+     *            Id of the domain for which resource recalculation to be doneDO
      * @param typeId
-     *            TODO
+     *            type of the resource for which recalculation to be done
+     * @param tag
+     *            tag for the resource type for which recalculation to be done
      * @return the updated/created resource counts
      */
+    List<? extends ResourceCount> recalculateResourceCount(Long accountId, Long domainId, Integer typeId, String tag);
     List<? extends ResourceCount> recalculateResourceCount(Long accountId, Long domainId, Integer typeId);
 
     /**
@@ -77,17 +94,18 @@ public interface ResourceLimitService {
      *            TODO
      * @return a list of limits that match the criteria
      */
-    public List<? extends ResourceLimit> searchForLimits(Long id, Long accountId, Long domainId, ResourceType resourceType, Long startIndex, Long pageSizeVal);
+    public List<? extends ResourceLimit> searchForLimits(Long id, Long accountId, Long domainId, ResourceType resourceType, String tag, Long startIndex, Long pageSizeVal);
 
     /**
      * Finds the resource limit for a specified account and type. If the account has an infinite limit, will check
      * the account's parent domain, and if that limit is also infinite, will return the ROOT domain's limit.
      *
      * @param account
      * @param type
+     * @param tag
      * @return resource limit
      */
-    public long findCorrectResourceLimitForAccount(Account account, ResourceType type);
+    public long findCorrectResourceLimitForAccount(Account account, ResourceType type, String tag);
 
     /**
      * This call should be used when we have already queried resource limit for an account. This is to handle
@@ -105,9 +123,10 @@ public interface ResourceLimitService {
      *
      * @param domain
      * @param type
+     * @param tag
      * @return resource limit
      */
-    public long findCorrectResourceLimitForDomain(Domain domain, ResourceType type);
+    public long findCorrectResourceLimitForDomain(Domain domain, ResourceType type, String tag);
 
     /**
      * Finds the default resource limit for a specified type.
@@ -122,9 +141,10 @@ public interface ResourceLimitService {
      *
      * @param domain
      * @param type
+     * @param tag
      * @return resource limit
      */
-    public long findCorrectResourceLimitForAccountAndDomain(Account account, Domain domain, ResourceType type);
+    public long findCorrectResourceLimitForAccountAndDomain(Account account, Domain domain, ResourceType type, String tag);
 
     /**
      * Increments the resource count
@@ -134,6 +154,7 @@ public interface ResourceLimitService {
      * @param delta
      */
     public void incrementResourceCount(long accountId, ResourceType type, Long... delta);
+    public void incrementResourceCountWithTag(long accountId, ResourceType type, String tag, Long... delta);
 
     /**
      * Decrements the resource count
@@ -143,6 +164,7 @@ public interface ResourceLimitService {
      * @param delta
      */
     public void decrementResourceCount(long accountId, ResourceType type, Long... delta);
+    public void decrementResourceCountWithTag(long accountId, ResourceType type, String tag, Long... delta);
 
     /**
      * Checks if a limit has been exceeded for an account
@@ -155,15 +177,17 @@ public interface ResourceLimitService {
      * @throws ResourceAllocationException
      */
     public void checkResourceLimit(Account account, ResourceCount.ResourceType type, long... count) throws ResourceAllocationException;
+    public void checkResourceLimitWithTag(Account account, ResourceCount.ResourceType type, String tag, long... count) throws ResourceAllocationException;
 
     /**
      * Gets the count of resources for a resource type and account
      *
      * @param account
      * @param type
+     * @param tag
      * @return count of resources
      */
-    public long getResourceCount(Account account, ResourceType type);
+    public long getResourceCount(Account account, ResourceType type, String tag);
 
     /**
      * Checks if a limit has been exceeded for an account if displayResource flag is on
@@ -208,15 +232,25 @@ public interface ResourceLimitService {
      */
     void decrementResourceCount(long accountId, ResourceType type, Boolean displayResource, Long... delta);
 
-    /**
-     * Adds a reservation that will be counted in subsequent calls to {count}getResourceCount{code} until {code}this[code}
-     * is closed. It will create a reservation record that will be counted when resource limits are checked.
-     * @param account The account for which the reservation is.
-     * @param displayResource whether this resource is shown to users at all (if not it is not counted to limits)
-     * @param type resource type
-     * @param delta amount to reserve (will not be <+ 0)
-     * @return a {code}AutoClosable{Code} object representing the resource the user needs
-     */
-    ResourceReservation getReservation(Account account, Boolean displayResource, ResourceType type, Long delta) throws ResourceAllocationException;
+    List<String> getResourceLimitHostTags();
+    List<String> getResourceLimitHostTags(ServiceOffering serviceOffering, VirtualMachineTemplate template);
+    List<String> getResourceLimitStorageTags();
+    List<String> getResourceLimitStorageTags(DiskOffering diskOffering);
+    void updateTaggedResourceLimitsAndCountsForAccounts(List<AccountResponse> responses, String tag);
+    void updateTaggedResourceLimitsAndCountsForDomains(List<DomainResponse> responses, String tag);
+    void checkVolumeResourceLimit(Account owner, Boolean display, Long size, DiskOffering diskOffering) throws ResourceAllocationException;
+    void incrementVolumeResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering);
+    void decrementVolumeResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering);
+    void incrementVolumePrimaryStorageResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering);
+    void decrementVolumePrimaryStorageResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering);
+    void checkVmResourceLimit(Account owner, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template) throws ResourceAllocationException;
+    void incrementVmResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template);
+    void decrementVmResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template);
+    void checkVmCpuResourceLimit(Account owner, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long cpu) throws ResourceAllocationException;
+    void incrementVmCpuResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long cpu);
+    void decrementVmCpuResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long cpu);
+    void checkVmMemoryResourceLimit(Account owner, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long memory) throws ResourceAllocationException;
+    void incrementVmMemoryResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long memory);
+    void decrementVmMemoryResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long memory);
 
 }

api/src/main/java/com/cloud/vm/NicProfile.java

@@ -442,6 +442,6 @@ public void deallocate() {
 
     @Override
     public String toString() {
-        return String.format("NicProfile %s", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "id", "vmId", "reservationId", "iPv4Address", "broadcastUri"));
+        return String.format("NicProfile %s", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "id", "vmId", "deviceId", "broadcastUri", "reservationId", "iPv4Address"));
     }
 }

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -285,6 +285,7 @@ public class ApiConstants {
     public static final String LAST_SERVER_STOP = "lastserverstop";
     public static final String LEVEL = "level";
     public static final String LENGTH = "length";
+    public static final String LIMIT = "limit";
     public static final String LIMIT_CPU_USE = "limitcpuuse";
     public static final String LIST_HOSTS = "listhosts";
     public static final String LOCK = "lock";
@@ -380,6 +381,7 @@ public class ApiConstants {
     public static final String RECONNECT = "reconnect";
     public static final String RECOVER = "recover";
     public static final String REQUIRES_HVM = "requireshvm";
+    public static final String RESOURCE_COUNT = "resourcecount";
     public static final String RESOURCE_NAME = "resourcename";
     public static final String RESOURCE_TYPE = "resourcetype";
     public static final String RESOURCE_TYPE_NAME = "resourcetypename";
@@ -420,8 +422,9 @@ public class ApiConstants {
     public static final String SNAPSHOT_POLICY_ID = "snapshotpolicyid";
     public static final String SNAPSHOT_TYPE = "snapshottype";
     public static final String SNAPSHOT_QUIESCEVM = "quiescevm";
-    public static final String SUPPORTS_STORAGE_SNAPSHOT = "supportsstoragesnapshot";
     public static final String SOURCE_ZONE_ID = "sourcezoneid";
+    public static final String SUITABLE_FOR_VM = "suitableforvirtualmachine";
+    public static final String SUPPORTS_STORAGE_SNAPSHOT = "supportsstoragesnapshot";
     public static final String START_DATE = "startdate";
     public static final String START_ID = "startid";
     public static final String START_IP = "startip";
@@ -449,6 +452,7 @@ public class ApiConstants {
     public static final String TIMEOUT = "timeout";
     public static final String TIMEZONE = "timezone";
     public static final String TIMEZONEOFFSET = "timezoneoffset";
+    public static final String TOTAL = "total";
     public static final String TOTAL_SUBNETS = "totalsubnets";
     public static final String TYPE = "type";
     public static final String TRUST_STORE = "truststore";
@@ -719,6 +723,7 @@ public class ApiConstants {
     public static final String POLICY_UUID = "policyuuid";
     public static final String RULE_UUID = "ruleuuid";
     public static final String DIRECTION = "direction";
+    public static final String TAGGED_RESOURCES = "taggedresources";
     public static final String TAG_UUID = "taguuid";
     public static final String TAG_TYPE = "tagtype";
     public static final String TAG_VALUE = "tagvalue";

api/src/main/java/org/apache/cloudstack/api/command/admin/domain/ListDomainsCmd.java

@@ -20,10 +20,6 @@
 import java.util.EnumSet;
 import java.util.List;
 
-import com.cloud.server.ResourceIcon;
-import com.cloud.server.ResourceTag;
-import org.apache.cloudstack.api.response.ResourceIconResponse;
-
 import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.ApiConstants.DomainDetails;
@@ -33,9 +29,13 @@
 import org.apache.cloudstack.api.command.user.UserCmd;
 import org.apache.cloudstack.api.response.DomainResponse;
 import org.apache.cloudstack.api.response.ListResponse;
+import org.apache.cloudstack.api.response.ResourceIconResponse;
+import org.apache.commons.collections.CollectionUtils;
 
 import com.cloud.domain.Domain;
 import com.cloud.exception.InvalidParameterValueException;
+import com.cloud.server.ResourceIcon;
+import com.cloud.server.ResourceTag;
 
 @APICommand(name = "listDomains", description = "Lists domains and provides detailed information for listed domains", responseObject = DomainResponse.class, responseView = ResponseView.Restricted, entityType = {Domain.class},
         requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
@@ -71,6 +71,9 @@ public class ListDomainsCmd extends BaseListCmd implements UserCmd {
             description = "flag to display the resource icon for domains")
     private Boolean showIcon;
 
+    @Parameter(name = ApiConstants.TAG, type = CommandType.STRING, description = "Tag for resource type to return usage", since = "4.20.0")
+    private String tag;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -110,10 +113,14 @@ public EnumSet<DomainDetails> getDetails() throws InvalidParameterValueException
         return dv;
     }
 
-    public Boolean getShowIcon() {
+    public boolean getShowIcon() {
         return showIcon != null ? showIcon : false;
     }
 
+    public String getTag() {
+        return tag;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////
@@ -128,12 +135,17 @@ public void execute() {
         ListResponse<DomainResponse> response = _queryService.searchForDomains(this);
         response.setResponseName(getCommandName());
         this.setResponseObject(response);
-        if (response != null && response.getCount() > 0 && getShowIcon()) {
-            updateDomainResponse(response.getResponses());
-        }
+        updateDomainResponse(response.getResponses());
     }
 
-    private void updateDomainResponse(List<DomainResponse> response) {
+    protected void updateDomainResponse(List<DomainResponse> response) {
+        if (CollectionUtils.isEmpty(response)) {
+            return;
+        }
+        _resourceLimitService.updateTaggedResourceLimitsAndCountsForDomains(response, getTag());
+        if (!getShowIcon()) {
+            return;
+        }
         for (DomainResponse domainResponse : response) {
             ResourceIcon resourceIcon = resourceIconManager.getByResourceTypeAndUuid(ResourceTag.ResourceObjectType.Domain, domainResponse.getId());
             if (resourceIcon == null) {