refactoring loading to persist YAML content in the DB and start implementing the listIacResourceTypes API

Author: bernardodemarco

engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42100to42200.java

@@ -19,7 +19,6 @@
 import com.cloud.utils.FileUtil;
 import com.cloud.utils.exception.CloudRuntimeException;
 
-import java.io.IOException;
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
@@ -31,7 +30,7 @@
 import java.util.List;
 
 public class Upgrade42100to42200 extends DbUpgradeAbstractImpl implements DbUpgrade, DbUpgradeSystemVmTemplate {
-    private static final Path NIMBLE_RESOURCE_TYPES_DIRECTORY = Paths.get("nimble", "resource-types");
+    private static final String NIMBLE_RESOURCE_TYPES_DIRECTORY = Paths.get("nimble", "resource-types").toString();
 
     @Override
     public String[] getUpgradableVersionRange() {
@@ -56,8 +55,8 @@ public InputStream[] getPrepareScripts() {
 
     @Override
     public void performDataMigration(Connection conn) {
-        updateSnapshotPolicyOwnership(conn);
-        updateBackupScheduleOwnership(conn);
+//        updateSnapshotPolicyOwnership(conn);
+//        updateBackupScheduleOwnership(conn);
         populateNimbleIacResourceTypes(conn);
     }
 
@@ -111,9 +110,9 @@ protected void updateBackupScheduleOwnership(Connection conn) {
     }
 
     protected void populateNimbleIacResourceTypes(Connection conn) {
-        String insertResourceTypeQuery = "INSERT INTO iac_templates_profile (uuid, name, element_content) VALUES (UUID(), ?, ?)";
+        String insertResourceTypeQuery = "INSERT INTO iac_resource_types (uuid, name, element_content) VALUES (UUID(), ?, ?)";
 
-        List<String> filePaths = FileUtil.getFilesPathsUnderResourceDirectory(NIMBLE_RESOURCE_TYPES_DIRECTORY.toString());
+        List<String> filePaths = FileUtil.getFilesPathsUnderResourceDirectory(NIMBLE_RESOURCE_TYPES_DIRECTORY);
         logger.info("Found the following NIMBLE's resource types files: [{}]. " +
                 "Each one of them will be iterated and its corresponding content will be inserted in the database.", filePaths);
         for (String filePath : filePaths) {
@@ -124,7 +123,7 @@ protected void populateNimbleIacResourceTypes(Connection conn) {
                 }
 
                 String resourceTypeElementContent = new String(inputStream.readAllBytes(), StandardCharsets.UTF_8);
-                preparedStatement.setString(1, filePath);
+                preparedStatement.setString(1, getIacResourceTypeNameFromFilePath(filePath));
                 preparedStatement.setString(2, resourceTypeElementContent);
                 preparedStatement.executeUpdate();
             } catch (SQLException exception) {
@@ -134,4 +133,20 @@ protected void populateNimbleIacResourceTypes(Connection conn) {
             }
         }
     }
+
+    protected String getIacResourceTypeNameFromFilePath(String filePath) {
+        String fileName = Path.of(filePath).getFileName().toString();
+        int fileExtensionDelimiterPosition = fileName.lastIndexOf('.');
+        String resourceNameInKebabCase = fileExtensionDelimiterPosition == -1 ?
+                fileName : fileName.substring(0, fileExtensionDelimiterPosition);
+        StringBuilder resourceNameInCamelCase = new StringBuilder();
+        for (String resourceNamePart : resourceNameInKebabCase.split("-")) {
+            if (!resourceNamePart.isEmpty()) {
+                resourceNameInCamelCase.append(Character.toUpperCase(resourceNamePart.charAt(0)))
+                        .append(resourceNamePart.substring(1));
+            }
+        }
+
+        return resourceNameInCamelCase.toString();
+    }
 }

engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql

@@ -20,80 +20,11 @@
 --;
 
 
--- health check status as enum
-CALL `cloud`.`IDEMPOTENT_CHANGE_COLUMN`('router_health_check', 'check_result', 'check_result', 'varchar(16) NOT NULL COMMENT "check executions result: SUCCESS, FAILURE, WARNING, UNKNOWN"');
-
--- Increase length of scripts_version column to 128 due to md5sum to sha512sum change
-CALL `cloud`.`IDEMPOTENT_CHANGE_COLUMN`('cloud.domain_router', 'scripts_version', 'scripts_version', 'VARCHAR(128)');
-
-CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.snapshot_policy','domain_id', 'BIGINT(20) DEFAULT NULL');
-CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.snapshot_policy','account_id', 'BIGINT(20) DEFAULT NULL');
-
-CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backup_schedule','domain_id', 'BIGINT(20) DEFAULT NULL');
-CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backup_schedule','account_id', 'BIGINT(20) DEFAULT NULL');
-
--- Increase the cache_mode column size from cloud.disk_offering table
-CALL `cloud`.`IDEMPOTENT_CHANGE_COLUMN`('cloud.disk_offering', 'cache_mode', 'cache_mode', 'varchar(18) DEFAULT "none" COMMENT "The disk cache mode to use for disks created with this offering"');
-
--- Add uuid column to ldap_configuration table
-CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.ldap_configuration', 'uuid', 'VARCHAR(40) NOT NULL');
-
--- Populate uuid for existing rows where uuid is NULL or empty
-UPDATE `cloud`.`ldap_configuration` SET uuid = UUID() WHERE uuid IS NULL OR uuid = '';
-
--- Add the column cross_zone_instance_creation to cloud.backup_repository. if enabled it means that new Instance can be created on all Zones from Backups on this Repository.
-CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backup_repository', 'cross_zone_instance_creation', 'TINYINT(1) DEFAULT NULL COMMENT ''Backup Repository can be used for disaster recovery on another zone''');
-
--- Updated display to false for password/token detail of the storage pool details
-UPDATE `cloud`.`storage_pool_details` SET display = 0 WHERE name LIKE '%password%';
-UPDATE `cloud`.`storage_pool_details` SET display = 0 WHERE name LIKE '%token%';
-
--- Add csi_enabled column to kubernetes_cluster table to indicate if the cluster is using csi or not
-CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster', 'csi_enabled', 'TINYINT(1) unsigned NOT NULL DEFAULT 0 COMMENT "true if kubernetes cluster is using csi, false otherwise" ');
-
--- VMware to KVM migration improvements
-CREATE TABLE IF NOT EXISTS `cloud`.`import_vm_task`(
-    `id` bigint unsigned NOT NULL auto_increment COMMENT 'id',
-    `uuid` varchar(40),
-    `zone_id` bigint unsigned NOT NULL COMMENT 'Zone ID',
-    `account_id` bigint unsigned NOT NULL COMMENT 'Account ID',
-    `user_id` bigint unsigned NOT NULL COMMENT 'User ID',
-    `vm_id` bigint unsigned COMMENT 'VM ID',
-    `display_name` varchar(255) COMMENT 'Display VM Name',
-    `vcenter` varchar(255) COMMENT 'VCenter',
-    `datacenter` varchar(255) COMMENT 'VCenter Datacenter name',
-    `source_vm_name` varchar(255) COMMENT 'Source VM name on vCenter',
-    `convert_host_id` bigint unsigned COMMENT 'Convert Host ID',
-    `import_host_id` bigint unsigned COMMENT 'Import Host ID',
-    `step` varchar(20) COMMENT 'Importing VM Task Step',
-    `state` varchar(20) COMMENT 'Importing VM Task State',
-    `description` varchar(255) COMMENT 'Importing VM Task Description',
-    `duration` bigint unsigned COMMENT 'Duration in milliseconds for the completed tasks',
-    `created` datetime NOT NULL COMMENT 'date created',
-    `updated` datetime COMMENT 'date updated if not null',
-    `removed` datetime COMMENT 'date removed if not null',
-    PRIMARY KEY (`id`),
-    CONSTRAINT `fk_import_vm_task__zone_id` FOREIGN KEY `fk_import_vm_task__zone_id` (`zone_id`) REFERENCES `data_center`(`id`) ON DELETE CASCADE,
-    CONSTRAINT `fk_import_vm_task__account_id` FOREIGN KEY `fk_import_vm_task__account_id` (`account_id`) REFERENCES `account`(`id`) ON DELETE CASCADE,
-    CONSTRAINT `fk_import_vm_task__user_id` FOREIGN KEY `fk_import_vm_task__user_id` (`user_id`) REFERENCES `user`(`id`) ON DELETE CASCADE,
-    CONSTRAINT `fk_import_vm_task__vm_id` FOREIGN KEY `fk_import_vm_task__vm_id` (`vm_id`) REFERENCES `vm_instance`(`id`) ON DELETE CASCADE,
-    CONSTRAINT `fk_import_vm_task__convert_host_id` FOREIGN KEY `fk_import_vm_task__convert_host_id` (`convert_host_id`) REFERENCES `host`(`id`) ON DELETE CASCADE,
-    CONSTRAINT `fk_import_vm_task__import_host_id` FOREIGN KEY `fk_import_vm_task__import_host_id` (`import_host_id`) REFERENCES `host`(`id`) ON DELETE CASCADE,
-    INDEX `i_import_vm_task__zone_id`(`zone_id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-
-CALL `cloud`.`INSERT_EXTENSION_IF_NOT_EXISTS`('MaaS', 'Baremetal Extension for Canonical MaaS written in Python', 'MaaS/maas.py');
-CALL `cloud`.`INSERT_EXTENSION_DETAIL_IF_NOT_EXISTS`('MaaS', 'orchestratorrequirespreparevm', 'true', 0);
-
-CALL `cloud`.`IDEMPOTENT_DROP_UNIQUE_KEY`('counter', 'uc_counter__provider__source__value');
-CALL `cloud`.`IDEMPOTENT_ADD_UNIQUE_KEY`('cloud.counter', 'uc_counter__provider__source__value__removed', '(provider, source, value, removed)');
-
--- NIMBLE - mudar para resource types talvez? iac_resource_types
-CREATE TABLE IF NOT EXISTS `cloud`.`iac_templates_profile` (
+-- NIMBLE
+CREATE TABLE IF NOT EXISTS `cloud`.`iac_resource_types` (
     `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT,
     `uuid` VARCHAR(40) UNIQUE,
     `name` VARCHAR(100) NOT NULL COMMENT 'Profile''s element name.',
---     `type` VARCHAR(100) NOT NULL COMMENT 'Profile''s element type. Valid values are: `CAPABILITY`, `RELATIONSHIP` and `NODE`.',
     `element_content` TEXT NOT NULL COMMENT 'Profile''s element content.',
     PRIMARY KEY (`id`)
 );

…e/resource_types/kubernetes-cluster.yaml …e/resource-types/kubernetes-cluster.yamlengine/schema/src/main/resources/nimble/resource_types/kubernetes-cluster.yaml renamed to engine/schema/src/main/resources/nimble/resource-types/kubernetes-cluster.yaml engine/schema/src/main/resources/nimble/resource_types/kubernetes-cluster.yaml renamed to engine/schema/src/main/resources/nimble/resource-types/kubernetes-cluster.yaml

@@ -18,22 +18,42 @@
 
 import org.apache.cloudstack.acl.RoleType;
 import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.BaseListCmd;
-import org.apache.cloudstack.api.response.IacResourceTypesResponse;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.response.IacResourceTypeResponse;
+import org.apache.cloudstack.api.response.ListResponse;
+import org.apache.cloudstack.persistence.iactemplatesprofile.IacResourceType;
 import org.apache.cloudstack.service.NimbleService;
 
 import javax.inject.Inject;
 
 @APICommand(name = "listIacResourceTypes",
-        description = "Lists available IaC resource types, such as TOSCA's node, relationship and capacity types.",
-        responseObject = IacResourceTypesResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
-        authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User})
+        description = "Lists all available IaC resource types, such as TOSCA node types.",
+        responseObject = IacResourceTypeResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
+        entityType = {IacResourceType.class}, authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User})
 public class ListIacResourceTypesCmd extends BaseListCmd {
     @Inject
     private NimbleService nimbleService;
 
+    @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = IacResourceTypeResponse.class, description = "The ID of the IaC resource type.")
+    private Long id;
+
+    @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "The name of the IaC resource type.")
+    private String name;
+
+    public Long getId() {
+        return id;
+    }
+
+    public String getName() {
+        return name;
+    }
+
     @Override
     public void execute() {
-        nimbleService.listIacResourceTypes();
+        ListResponse<IacResourceTypeResponse> response = nimbleService.listIacResourceTypes(this);
+        response.setResponseName(getCommandName());
+        setResponseObject(response);
     }
 }

…ources/nimble/resource_types/volume.yaml …ources/nimble/resource-types/volume.yamlengine/schema/src/main/resources/nimble/resource_types/volume.yaml renamed to engine/schema/src/main/resources/nimble/resource-types/volume.yaml engine/schema/src/main/resources/nimble/resource_types/volume.yaml renamed to engine/schema/src/main/resources/nimble/resource-types/volume.yaml

@@ -20,5 +20,5 @@
 import org.apache.cloudstack.api.EntityReference;
 
 @EntityReference(value = {})
-public class IacResourceTypesResponse extends BaseResponseWithAnnotations {
+public class IacResourceTypeResponse extends BaseResponseWithAnnotations {
 }

plugins/iac/nimble/src/main/java/org/apache/cloudstack/api/command/ListIacResourceTypesCmd.java

@@ -14,10 +14,7 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
-package org.apache.cloudstack.persistence.iactemplatesprofile;
-
-import com.cloud.utils.db.GenericDao;
-
-public interface IacTemplatesProfileDao extends GenericDao<IacTemplatesProfileVO, Long> {
+package org.apache.cloudstack.api.response;
 
+public class NimbleResponseBuilder {
 }

…i/response/IacResourceTypesResponse.java …pi/response/IacResourceTypeResponse.javaplugins/iac/nimble/src/main/java/org/apache/cloudstack/api/response/IacResourceTypesResponse.java renamed to plugins/iac/nimble/src/main/java/org/apache/cloudstack/api/response/IacResourceTypeResponse.java plugins/iac/nimble/src/main/java/org/apache/cloudstack/api/response/IacResourceTypesResponse.java renamed to plugins/iac/nimble/src/main/java/org/apache/cloudstack/api/response/IacResourceTypeResponse.java

@@ -19,13 +19,7 @@
 import org.apache.cloudstack.api.Identity;
 import org.apache.cloudstack.api.InternalIdentity;
 
-public interface IacTemplatesProfile extends Identity, InternalIdentity {
-    enum Type {
-        CAPABILITY, RELATIONSHIP, NODE
-    }
-
-    long getId();
+public interface IacResourceType extends Identity, InternalIdentity {
     String getName();
-    Type getType();
-    String getElementContentFilePath();
+    String getElementContent();
 }

…latesprofile/IacTemplatesProfileDao.java …/api/response/NimbleResponseBuilder.javaplugins/iac/nimble/src/main/java/org/apache/cloudstack/persistence/iactemplatesprofile/IacTemplatesProfileDao.java renamed to plugins/iac/nimble/src/main/java/org/apache/cloudstack/api/response/NimbleResponseBuilder.java plugins/iac/nimble/src/main/java/org/apache/cloudstack/persistence/iactemplatesprofile/IacTemplatesProfileDao.java renamed to plugins/iac/nimble/src/main/java/org/apache/cloudstack/api/response/NimbleResponseBuilder.java

@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.persistence.iactemplatesprofile;
+
+import com.cloud.utils.Pair;
+import com.cloud.utils.db.GenericDao;
+
+import java.util.List;
+
+public interface IacResourceTypeDao extends GenericDao<IacResourceTypeVO, Long> {
+    Pair<List<IacResourceTypeVO>, Integer> listIacResourceTypes(Long id, String name, String keyword, Long pageSizeVal, Long startIndex);
+}

…emplatesprofile/IacTemplatesProfile.java …iactemplatesprofile/IacResourceType.javaplugins/iac/nimble/src/main/java/org/apache/cloudstack/persistence/iactemplatesprofile/IacTemplatesProfile.java renamed to plugins/iac/nimble/src/main/java/org/apache/cloudstack/persistence/iactemplatesprofile/IacResourceType.java plugins/iac/nimble/src/main/java/org/apache/cloudstack/persistence/iactemplatesprofile/IacTemplatesProfile.java renamed to plugins/iac/nimble/src/main/java/org/apache/cloudstack/persistence/iactemplatesprofile/IacResourceType.java

@@ -20,5 +20,5 @@
 import org.springframework.stereotype.Component;
 
 @Component
-public class IacTemplatesProfileDaoImpl extends GenericDaoBase<IacTemplatesProfileVO, Long> implements IacTemplatesProfileDao {
+public class IacResourceTypeDaoImpl extends GenericDaoBase<IacResourceTypeVO, Long> implements IacResourceTypeDao {
 }