Resource reservation framework (#6694)

This PR addresses parallel resource allocation as a generalization of the problem and solution described in #6644. Instead of the Global lock on the resources a reservation record is created which is added in the resource check count in the ResourceLimitService/ResourceLimitManagerImpl. As a convenience a CheckedReservation is created. This is an implementation of AutoClosable and can be used as a guard in a try-with-resource fashion. The close method of the CheckedReservation wil delete the reservation record.

Co-authored-by: Boris Stoyanov - a.k.a Bobby <bss.stoyanov@gmail.com>

Author: DaanHoogland

api/src/main/java/com/cloud/user/ResourceLimitService.java

@@ -24,6 +24,7 @@
 import com.cloud.domain.Domain;
 import com.cloud.exception.ResourceAllocationException;
 import org.apache.cloudstack.framework.config.ConfigKey;
+import org.apache.cloudstack.user.ResourceReservation;
 
 public interface ResourceLimitService {
 
@@ -91,7 +92,7 @@ public interface ResourceLimitService {
     /**
      * This call should be used when we have already queried resource limit for an account. This is to handle
      * some corner cases where queried limit may be null.
-     * @param accountType
+     * @param accountId
      * @param limit
      * @param type
      * @return
@@ -102,7 +103,7 @@ public interface ResourceLimitService {
      * Finds the resource limit for a specified domain and type. If the domain has an infinite limit, will check
      * up the domain hierarchy
      *
-     * @param account
+     * @param domain
      * @param type
      * @return resource limit
      */
@@ -197,4 +198,16 @@ public interface ResourceLimitService {
      * @param delta
      */
     void decrementResourceCount(long accountId, ResourceType type, Boolean displayResource, Long... delta);
+
+    /**
+     * Adds a reservation that will be counted in subsequent calls to {count}getResourceCount{code} until {code}this[code}
+     * is closed. It will create a reservation record that will be counted when resource limits are checked.
+     * @param account The account for which the reservation is.
+     * @param displayResource whether this resource is shown to users at all (if not it is not counted to limits)
+     * @param type resource type
+     * @param delta amount to reserve (will not be <+ 0)
+     * @return a {code}AutoClosable{Code} object representing the resource the user needs
+     */
+    ResourceReservation getReservation(Account account, Boolean displayResource, ResourceType type, Long delta) throws ResourceAllocationException;
+
 }

…ache/cloudstack/acl/APILimitChecker.java …cloudstack/user/ResourceReservation.javaapi/src/main/java/org/apache/cloudstack/acl/APILimitChecker.java renamed to api/src/main/java/org/apache/cloudstack/user/ResourceReservation.java api/src/main/java/org/apache/cloudstack/acl/APILimitChecker.java renamed to api/src/main/java/org/apache/cloudstack/user/ResourceReservation.java

@@ -1,3 +1,4 @@
+//
 // Licensed to the Apache Software Foundation (ASF) under one
 // or more contributor license agreements.  See the NOTICE file
 // distributed with this work for additional information
@@ -14,17 +15,23 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
-package org.apache.cloudstack.acl;
-
-import org.apache.cloudstack.api.ServerApiException;
+//
+package org.apache.cloudstack.user;
 
-import com.cloud.user.Account;
-import com.cloud.utils.component.Adapter;
+import com.cloud.configuration.Resource;
+import org.apache.cloudstack.api.InternalIdentity;
 
 /**
- * APILimitChecker checks if we should block an API request based on pre-set account based api limit.
+ * an interface defining an {code}AutoClosable{code} reservation object
  */
-public interface APILimitChecker extends Adapter {
-    // Interface for checking if the account is over its api limit
-    void checkLimit(Account account) throws ServerApiException;
+public interface
+ResourceReservation extends InternalIdentity {
+
+    Long getAccountId();
+
+    Long getDomainId();
+
+    Resource.ResourceType getResourceType();
+
+    Long getReservedAmount();
 }

core/src/test/java/com/cloud/storage/template/LocalTemplateDownloaderTest.java

@@ -19,7 +19,7 @@
 
 package com.cloud.storage.template;
 
-import static org.junit.Assert.fail;
+import static org.junit.Assert.assertTrue;
 
 import java.io.File;
 
@@ -33,9 +33,7 @@ public void localTemplateDownloaderTest() throws Exception {
         String url = new File("pom.xml").toURI().toURL().toString();
         TemplateDownloader td = new LocalTemplateDownloader(null, url, System.getProperty("java.io.tmpdir"), TemplateDownloader.DEFAULT_MAX_TEMPLATE_SIZE_IN_BYTES, null);
         long bytes = td.download(true, null);
-        if (!(bytes > 0)) {
-            fail("Failed download");
-        }
+        assertTrue("Failed download", bytes > 0);
     }
 
 }

engine/schema/src/main/java/org/apache/cloudstack/reservation/ReservationVO.java

@@ -0,0 +1,90 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package org.apache.cloudstack.reservation;
+
+import com.cloud.configuration.Resource;
+import org.apache.cloudstack.user.ResourceReservation;
+import com.cloud.utils.exception.CloudRuntimeException;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import javax.persistence.Table;
+
+@Entity
+@Table(name = "resource_reservation")
+public class ReservationVO implements ResourceReservation {
+
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    @Column(name = "id")
+    Long id;
+
+    @Column(name = "account_id")
+    long accountId;
+
+    @Column(name = "domain_id")
+    long domainId;
+
+    @Column(name = "resource_type", nullable = false)
+    Resource.ResourceType resourceType;
+
+    @Column(name = "amount")
+    long amount;
+
+    protected ReservationVO()
+    {}
+
+    public ReservationVO(Long accountId, Long domainId, Resource.ResourceType resourceType, Long delta) {
+        if (delta == null || delta <= 0) {
+            throw new CloudRuntimeException("resource reservations can not be made for no resources");
+        }
+        this.accountId = accountId;
+        this.domainId = domainId;
+        this.resourceType = resourceType;
+        this.amount = delta;
+    }
+
+    @Override
+    public long getId() {
+        return this.id;
+    }
+
+    @Override
+    public Long getAccountId() {
+        return accountId;
+    }
+
+    @Override
+    public Long getDomainId() {
+        return domainId;
+    }
+
+    @Override
+    public Resource.ResourceType getResourceType() {
+        return resourceType;
+    }
+
+    @Override
+    public Long getReservedAmount() {
+        return amount;
+    }
+}

engine/schema/src/main/java/org/apache/cloudstack/reservation/dao/ReservationDao.java

@@ -0,0 +1,28 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package org.apache.cloudstack.reservation.dao;
+
+import com.cloud.configuration.Resource;
+import org.apache.cloudstack.reservation.ReservationVO;
+import com.cloud.utils.db.GenericDao;
+
+public interface ReservationDao extends GenericDao<ReservationVO, Long> {
+    long getAccountReservation(Long account, Resource.ResourceType resourceType);
+    long getDomainReservation(Long domain, Resource.ResourceType resourceType);
+}

engine/schema/src/main/java/org/apache/cloudstack/reservation/dao/ReservationDaoImpl.java

@@ -0,0 +1,75 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package org.apache.cloudstack.reservation.dao;
+
+import com.cloud.configuration.Resource;
+import com.cloud.utils.db.GenericDaoBase;
+import com.cloud.utils.db.SearchBuilder;
+import com.cloud.utils.db.SearchCriteria;
+import org.apache.cloudstack.reservation.ReservationVO;
+
+import java.util.List;
+
+public class ReservationDaoImpl extends GenericDaoBase<ReservationVO, Long> implements ReservationDao {
+
+    private static final String RESOURCE_TYPE = "resourceType";
+    private static final String ACCOUNT_ID = "accountId";
+    private static final String DOMAIN_ID = "domainId";
+    private final SearchBuilder<ReservationVO> listAccountAndTypeSearch;
+
+    private final SearchBuilder<ReservationVO> listDomainAndTypeSearch;
+
+    public ReservationDaoImpl() {
+        listAccountAndTypeSearch = createSearchBuilder();
+        listAccountAndTypeSearch.and(ACCOUNT_ID, listAccountAndTypeSearch.entity().getAccountId(), SearchCriteria.Op.EQ);
+        listAccountAndTypeSearch.and(RESOURCE_TYPE, listAccountAndTypeSearch.entity().getResourceType(), SearchCriteria.Op.EQ);
+        listAccountAndTypeSearch.done();
+
+        listDomainAndTypeSearch = createSearchBuilder();
+        listDomainAndTypeSearch.and(DOMAIN_ID, listDomainAndTypeSearch.entity().getDomainId(), SearchCriteria.Op.EQ);
+        listDomainAndTypeSearch.and(RESOURCE_TYPE, listDomainAndTypeSearch.entity().getResourceType(), SearchCriteria.Op.EQ);
+        listDomainAndTypeSearch.done();
+    }
+
+    @Override
+    public long getAccountReservation(Long accountId, Resource.ResourceType resourceType) {
+        long total = 0;
+        SearchCriteria<ReservationVO> sc = listAccountAndTypeSearch.create();
+        sc.setParameters(ACCOUNT_ID, accountId);
+        sc.setParameters(RESOURCE_TYPE, resourceType);
+        List<ReservationVO> reservations = listBy(sc);
+        for (ReservationVO reservation : reservations) {
+            total += reservation.getReservedAmount();
+        }
+        return total;
+    }
+
+    @Override
+    public long getDomainReservation(Long domainId, Resource.ResourceType resourceType) {
+        long total = 0;
+        SearchCriteria<ReservationVO> sc = listAccountAndTypeSearch.create();
+        sc.setParameters(DOMAIN_ID, domainId);
+        sc.setParameters(RESOURCE_TYPE, resourceType);
+        List<ReservationVO> reservations = listBy(sc);
+        for (ReservationVO reservation : reservations) {
+            total += reservation.getReservedAmount();
+        }
+        return total;
+    }
+}

engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml

@@ -175,6 +175,7 @@
   <bean id="projectJoinDaoImpl" class="com.cloud.api.query.dao.ProjectJoinDaoImpl" />
   <bean id="regionDaoImpl" class="org.apache.cloudstack.region.dao.RegionDaoImpl" />
   <bean id="remoteAccessVpnDaoImpl" class="com.cloud.network.dao.RemoteAccessVpnDaoImpl" />
+  <bean id="reservationDao" class="org.apache.cloudstack.reservation.dao.ReservationDaoImpl" />
   <bean id="resourceCountDaoImpl" class="com.cloud.configuration.dao.ResourceCountDaoImpl" />
   <bean id="resourceIconDaoImpl" class="com.cloud.resource.icon.dao.ResourceIconDaoImpl" />
   <bean id="resourceLimitDaoImpl" class="com.cloud.configuration.dao.ResourceLimitDaoImpl" />

engine/schema/src/main/resources/META-INF/db/schema-41710to41800.sql

@@ -27,6 +27,18 @@ WHERE so.default_use = 1 AND so.vm_type IN ('domainrouter', 'secondarystoragevm'
 ALTER TABLE `cloud`.`load_balancing_rules`
 ADD cidr_list VARCHAR(4096);
 
+-- savely add resources in parallel
+-- PR#5984 Create table to persist VM stats.
+DROP TABLE IF EXISTS `cloud`.`resource_reservation`;
+CREATE TABLE `cloud`.`resource_reservation` (
+  `id` bigint unsigned NOT NULL auto_increment COMMENT 'id',
+  `account_id` bigint unsigned NOT NULL,
+  `domain_id` bigint unsigned NOT NULL,
+  `resource_type` varchar(255) NOT NULL,
+  `amount` bigint unsigned NOT NULL,
+  PRIMARY KEY (`id`)
+  ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
 -- Alter networks table to add ip6dns1 and ip6dns2
 ALTER TABLE `cloud`.`networks`
     ADD COLUMN `ip6dns1` varchar(255) DEFAULT NULL COMMENT 'first IPv6 DNS for the network' AFTER `dns2`,