Quota custom tariffs

api/src/main/java/org/apache/cloudstack/api/ApiArgValidator.java

@@ -18,6 +18,18 @@
 package org.apache.cloudstack.api;
 
 public enum ApiArgValidator {
-    NotNullOrEmpty, // does StringUtils.isEmpty check
-    PositiveNumber, // does != null and > 0 check
+    /**
+     * Validates if the parameter is null or empty with the method {@link Strings#isNullOrEmpty(String)}.
+     */
+    NotNullOrEmpty,
+
+    /**
+     * Validates if the parameter is different from null (parameter != null) and greater than zero (parameter > 0).
+     */
+    PositiveNumber,
+
+    /**
+     * Validates if the parameter is an UUID with the method {@link UuidUtils#validateUUID(String)}.
+     */
+    UuidString,
 }

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -21,6 +21,7 @@ public class ApiConstants {
     public static final String ACCOUNTS = "accounts";
     public static final String ACCOUNT_TYPE = "accounttype";
     public static final String ACCOUNT_ID = "accountid";
+    public static final String ACTIVATION_RULE = "activationrule";
     public static final String ACTIVITY = "activity";
     public static final String ADAPTER_TYPE = "adaptertype";
     public static final String ADDRESS = "address";

api/src/main/java/org/apache/cloudstack/usage/UsageUnitTypes.java

@@ -0,0 +1,52 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.usage;
+
+public enum UsageUnitTypes {
+    ComputeMonth ("Compute-Month"),
+    IpMonth ("IP-Month"),
+    Gb ("GB"),
+    GbMonth ("GB-Month"),
+    PolicyMonth ("Policy-Month");
+
+    private final String description;
+
+    private UsageUnitTypes(String description) {
+        this.description = description;
+    }
+
+    @Override
+    public String toString() {
+        return description;
+    }
+
+    /**
+     * Retrieves the UsageUnitTypes according to the parameter.<br/><br/>
+     * If there are no UsageUnitTypes with the description, it will try to retrieve it with {@link UsageUnitTypes#valueOf(String)} and will throw an
+     * {@link IllegalArgumentException} if it not exist.
+     */
+    public static UsageUnitTypes getByDescription(String description) {
+        for (UsageUnitTypes type : UsageUnitTypes.values()) {
+            if (type.toString().equals(description)) {
+                return type;
+            }
+        }
+
+        return UsageUnitTypes.valueOf(description);
+    }
+}

api/src/test/java/org/apache/cloudstack/usage/UsageUnitTypesTest.java

@@ -0,0 +1,53 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.usage;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.junit.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public class UsageUnitTypesTest {
+
+    private List<UsageUnitTypes> usageUnitTypes = Arrays.asList(UsageUnitTypes.values());
+
+    @Test
+    public void getByDescriptionTestAllTheDescriptionsMustReturnUsageUnitTypes() {
+        usageUnitTypes.forEach(type -> {
+            UsageUnitTypes usageUnitType = UsageUnitTypes.getByDescription(type.toString());
+            Assert.assertEquals(type, usageUnitType);
+        });
+    }
+
+    @Test
+    public void getByDescriptionTestAllTheConstantNamesMustReturnUsageUnitTypes() {
+        usageUnitTypes.forEach(type -> {
+            UsageUnitTypes usageUnitType = UsageUnitTypes.getByDescription(type.name());
+            Assert.assertEquals(type, usageUnitType);
+        });
+    }
+
+    @Test (expected = IllegalArgumentException.class)
+    public void getByDescriptionTestPassWrongTypeOrDescriptionAndThrowsIllegalArgumentException() {
+        UsageUnitTypes.getByDescription("test");
+    }
+}

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotInfo.java

@@ -18,10 +18,16 @@
 
 import java.util.List;
 
+import org.apache.cloudstack.framework.config.ConfigKey;
+
 import com.cloud.storage.Snapshot;
 import com.cloud.utils.exception.CloudRuntimeException;
 
 public interface SnapshotInfo extends DataObject, Snapshot {
+    ConfigKey<Boolean> BackupSnapshotAfterTakingSnapshot = new ConfigKey<Boolean>(Boolean.class, "snapshot.backup.to.secondary",  "Snapshots", "true", "Indicates whether to always"
+            + " backup primary storage snapshot to secondary storage. Keeping snapshots only on Primary storage is applicable for KVM + Ceph only.", false, ConfigKey.Scope.Global,
+            null);
+
     SnapshotInfo getParent();
 
     String getPath();

engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java

@@ -132,7 +132,6 @@
 import com.cloud.storage.dao.VMTemplateDetailsDao;
 import com.cloud.storage.dao.VolumeDao;
 import com.cloud.storage.dao.VolumeDetailsDao;
-import com.cloud.storage.snapshot.SnapshotManager;
 import com.cloud.template.TemplateManager;
 import com.cloud.template.VirtualMachineTemplate;
 import com.cloud.user.Account;
@@ -564,7 +563,7 @@ public VolumeInfo createVolumeFromSnapshot(Volume volume, Snapshot snapshot, Use
     }
 
     private SnapshotInfo backupSnapshotIfNeeded(Snapshot snapshot, DataStoreRole dataStoreRole, SnapshotInfo snapInfo) {
-        boolean backupSnapToSecondary = SnapshotManager.BackupSnapshotAfterTakingSnapshot.value() == null || SnapshotManager.BackupSnapshotAfterTakingSnapshot.value();
+        boolean backupSnapToSecondary = SnapshotInfo.BackupSnapshotAfterTakingSnapshot.value() == null || SnapshotInfo.BackupSnapshotAfterTakingSnapshot.value();
 
         StoragePoolVO srcPool = _storagePoolDao.findById(snapInfo.getBaseVolume().getPoolId());
         // We need to copy the snapshot onto secondary.

engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41610to41700.java

@@ -18,10 +18,23 @@
 
 import com.cloud.upgrade.SystemVmTemplateRegistration;
 import com.cloud.utils.exception.CloudRuntimeException;
+
+import org.apache.cloudstack.api.response.UsageTypeResponse;
+import org.apache.cloudstack.usage.UsageTypes;
+import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
+import org.apache.commons.lang3.time.DateUtils;
 import org.apache.log4j.Logger;
 
 import java.io.InputStream;
 import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
 
 public class Upgrade41610to41700 implements DbUpgrade, DbUpgradeSystemVmTemplate {
 
@@ -56,6 +69,7 @@ public InputStream[] getPrepareScripts() {
 
     @Override
     public void performDataMigration(Connection conn) {
+        convertQuotaTariffsToNewParadigm(conn);
     }
 
     @Override
@@ -83,4 +97,100 @@ public void updateSystemVmTemplates(Connection conn) {
             throw new CloudRuntimeException("Failed to find / register SystemVM template(s)");
         }
     }
+
+    protected void convertQuotaTariffsToNewParadigm(Connection conn) {
+        LOG.info("Converting quota tariffs to new paradigm.");
+
+        List<UsageTypeResponse> usageTypeResponses = UsageTypes.listUsageTypes();
+
+        for (UsageTypeResponse usageTypeResponse : usageTypeResponses) {
+            Integer usageType = usageTypeResponse.getUsageType();
+
+            String tariffTypeDescription = ReflectionToStringBuilderUtils.reflectOnlySelectedFields(usageTypeResponse, "description", "usageType");
+
+            LOG.info(String.format("Converting quota tariffs of type %s to new paradigm.", tariffTypeDescription));
+
+            for (boolean previousTariff : Arrays.asList(true, false)) {
+                Map<Long, Date> tariffs = selectTariffs(conn, usageType, previousTariff, tariffTypeDescription);
+
+                int tariffsSize = tariffs.size();
+                if (tariffsSize <  2) {
+                    LOG.info(String.format("Quota tariff of type %s has [%s] %s register(s). Tariffs with less than 2 register do not need to be converted to new paradigm.",
+                            tariffTypeDescription, tariffsSize, previousTariff ? "previous of current" : "next to current"));
+                    continue;
+                }
+
+                executeUpdateQuotaTariffSetEndDateAndRemoved(conn, usageType, tariffs, previousTariff, tariffTypeDescription);
+            }
+        }
+    }
+
+    protected Map<Long, Date> selectTariffs(Connection conn, Integer usageType, boolean previousTariff, String tariffTypeDescription) {
+        Map<Long, Date> quotaTariffs = new LinkedHashMap<>();
+
+        String selectQuotaTariffs = String.format("SELECT id, effective_on FROM cloud_usage.quota_tariff WHERE %s AND usage_type = ? ORDER BY effective_on, updated_on;",
+                previousTariff ? "usage_name = name" : "removed is null");
+
+        LOG.info(String.format("Selecting %s quota tariffs of type [%s] according to SQL [%s].", previousTariff ? "previous of current" : "next to current",
+                tariffTypeDescription, selectQuotaTariffs));
+
+        try (PreparedStatement pstmt = conn.prepareStatement(selectQuotaTariffs)) {
+            pstmt.setInt(1, usageType);
+
+            try (ResultSet result = pstmt.executeQuery()) {
+                while (result.next()) {
+                    quotaTariffs.put(result.getLong("id"), result.getDate("effective_on"));
+                }
+            }
+            return quotaTariffs;
+        } catch (SQLException e) {
+            String message = String.format("Unable to retrieve %s quota tariffs of type [%s] due to [%s].", previousTariff ? "previous" : "next", tariffTypeDescription,
+                    e.getMessage());
+            LOG.error(message, e);
+            throw new CloudRuntimeException(message, e);
+        }
+    }
+
+    protected void executeUpdateQuotaTariffSetEndDateAndRemoved(Connection conn, Integer usageType, Map<Long, Date> tariffs, boolean setRemoved, String tariffTypeDescription) {
+        String updateQuotaTariff = String.format("UPDATE cloud_usage.quota_tariff SET end_date = ? %s WHERE id = ?;", setRemoved ? ", removed = ?" : "");
+
+        Object[] ids = tariffs.keySet().toArray();
+
+        LOG.info(String.format("Updating %s registers of %s quota tariffs of type [%s] with SQL [%s].", tariffs.size() -1, setRemoved ? "previous of current" :
+            "next to current", tariffTypeDescription, updateQuotaTariff));
+
+        for (int i = 0; i < tariffs.size() - 1; i++) {
+            Long id = Long.valueOf(String.valueOf(ids[i]));
+            Long nextId = Long.valueOf(String.valueOf(ids[i + 1]));
+
+            Date endDate = tariffs.get(nextId);
+
+            if (!DateUtils.isSameDay(endDate, tariffs.get(id))) {
+                endDate = DateUtils.addDays(endDate, -1);
+            }
+
+            try (PreparedStatement pstmt = conn.prepareStatement(updateQuotaTariff)) {
+                java.sql.Date sqlEndDate = new java.sql.Date(endDate.getTime());
+                pstmt.setDate(1, sqlEndDate);
+
+                String updateRemoved = "";
+                if (setRemoved) {
+                    pstmt.setDate(2, sqlEndDate);
+                    pstmt.setLong(3, id);
+
+                    updateRemoved = String.format("and \"removed\" to [%s] ", sqlEndDate);
+                } else {
+                    pstmt.setLong(2, id);
+                }
+
+                LOG.info(String.format("Updating \"end_date\" to [%s] %sof quota tariff with ID [%s].", sqlEndDate, updateRemoved, id));
+                pstmt.executeUpdate();
+            } catch (SQLException e) {
+                String message = String.format("Unable to update \"end_date\" %s of quota tariffs of usage type [%s] due to [%s].", setRemoved ? "and \"removed\"" : "",
+                        usageType, e.getMessage());
+                LOG.error(message, e);
+                throw new CloudRuntimeException(message, e);
+            }
+        }
+    }
 }

engine/schema/src/main/java/com/cloud/usage/UsageVO.java

@@ -29,6 +29,7 @@
 
 import org.apache.cloudstack.api.InternalIdentity;
 import org.apache.cloudstack.usage.Usage;
+import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
 
 @Entity
 @Table(name = "cloud_usage")
@@ -396,4 +397,9 @@ public void setVirtualSize(Long virtualSize) {
     public void setHidden(boolean hidden) {
         this.isHidden = hidden;
     }
+
+    @Override
+    public String toString() {
+        return ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "id", "usageId", "usageType", "startDate", "endDate");
+    }
 }

engine/schema/src/main/java/com/cloud/usage/dao/UsageDao.java

@@ -25,6 +25,7 @@
 import com.cloud.utils.db.GenericDao;
 import com.cloud.utils.db.SearchCriteria;
 
+import java.util.Date;
 import java.util.List;
 
 public interface UsageDao extends GenericDao<UsageVO, Long> {
@@ -58,5 +59,7 @@ public interface UsageDao extends GenericDao<UsageVO, Long> {
 
     UsageVO persistUsage(final UsageVO usage);
 
-    Pair<List<? extends UsageVO>, Integer> getUsageRecordsPendingQuotaAggregation(long accountId, long domainId);
+    Pair<List<UsageVO>, Integer> listUsageRecordsPendingForQuotaAggregation(long accountId, long domainId);
+
+    List<Pair<String, String>> listAccountResourcesInThePeriod(long accountId, int usageType, Date startDate, Date endDate);
 }