refactoring loading to persist YAML content in the DB

Author: bernardodemarco

engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42100to42200.java

@@ -16,15 +16,22 @@
 // under the License.
 package com.cloud.upgrade.dao;
 
+import com.cloud.utils.FileUtil;
 import com.cloud.utils.exception.CloudRuntimeException;
 
+import java.io.IOException;
 import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
+import java.util.List;
 
 public class Upgrade42100to42200 extends DbUpgradeAbstractImpl implements DbUpgrade, DbUpgradeSystemVmTemplate {
+    private static final Path NIMBLE_RESOURCE_TYPES_DIRECTORY = Paths.get("nimble", "resource-types");
 
     @Override
     public String[] getUpgradableVersionRange() {
@@ -51,6 +58,7 @@ public InputStream[] getPrepareScripts() {
     public void performDataMigration(Connection conn) {
         updateSnapshotPolicyOwnership(conn);
         updateBackupScheduleOwnership(conn);
+        populateNimbleIacResourceTypes(conn);
     }
 
     protected void updateSnapshotPolicyOwnership(Connection conn) {
@@ -99,5 +107,31 @@ protected void updateBackupScheduleOwnership(Connection conn) {
         } catch (SQLException e) {
             throw new CloudRuntimeException("Unable to update backup_schedule table with account_id and domain_id", e);
         }
+
+    }
+
+    protected void populateNimbleIacResourceTypes(Connection conn) {
+        String insertResourceTypeQuery = "INSERT INTO iac_templates_profile (uuid, name, element_content) VALUES (UUID(), ?, ?)";
+
+        List<String> filePaths = FileUtil.getFilesPathsUnderResourceDirectory(NIMBLE_RESOURCE_TYPES_DIRECTORY.toString());
+        logger.info("Found the following NIMBLE's resource types files: [{}]. " +
+                "Each one of them will be iterated and its corresponding content will be inserted in the database.", filePaths);
+        for (String filePath : filePaths) {
+            try (InputStream inputStream = Thread.currentThread().getContextClassLoader().getResourceAsStream(filePath);
+                 PreparedStatement preparedStatement = conn.prepareStatement(insertResourceTypeQuery)) {
+                if (inputStream == null) {
+                    throw new Exception(String.format("[%s] file's input stream is [null].", filePath));
+                }
+
+                String resourceTypeElementContent = new String(inputStream.readAllBytes(), StandardCharsets.UTF_8);
+                preparedStatement.setString(1, filePath);
+                preparedStatement.setString(2, resourceTypeElementContent);
+                preparedStatement.executeUpdate();
+            } catch (SQLException exception) {
+                logger.warn("Unable to insert resource type [{}] in the database. Skipping it.", filePath, exception);
+            } catch (Exception exception) {
+                logger.warn("Unable to read file: [{}]. Skipping it.", filePath, exception);
+            }
+        }
     }
 }

engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql

@@ -88,12 +88,12 @@ CALL `cloud`.`INSERT_EXTENSION_DETAIL_IF_NOT_EXISTS`('MaaS', 'orchestratorrequir
 CALL `cloud`.`IDEMPOTENT_DROP_UNIQUE_KEY`('counter', 'uc_counter__provider__source__value');
 CALL `cloud`.`IDEMPOTENT_ADD_UNIQUE_KEY`('cloud.counter', 'uc_counter__provider__source__value__removed', '(provider, source, value, removed)');
 
--- NIMBLE
+-- NIMBLE - mudar para resource types talvez? iac_resource_types
 CREATE TABLE IF NOT EXISTS `cloud`.`iac_templates_profile` (
     `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT,
     `uuid` VARCHAR(40) UNIQUE,
     `name` VARCHAR(100) NOT NULL COMMENT 'Profile''s element name.',
-    `type` VARCHAR(100) NOT NULL COMMENT 'Profile''s element type. Valid values are: `CAPABILITY`, `RELATIONSHIP` and `NODE`.',
-    `element_content_file_path` VARCHAR(255) NOT NULL COMMENT 'Profile''s element content file path.',
+--     `type` VARCHAR(100) NOT NULL COMMENT 'Profile''s element type. Valid values are: `CAPABILITY`, `RELATIONSHIP` and `NODE`.',
+    `element_content` TEXT NOT NULL COMMENT 'Profile''s element content.',
     PRIMARY KEY (`id`)
 );

engine/schema/src/main/resources/nimble/resource_types/kubernetes-cluster.yaml

@@ -0,0 +1,86 @@
+tosca_definitions_version: tosca_2_0
+
+description: >
+  KubernetesCluster node type definition.
+
+node_types:
+  KubernetesCluster:
+    derived_from: Root
+    properties:
+      name:
+        type: string
+        description: The name of the Kubernetes cluster.
+        required: true
+      zone-id:
+        type: string
+        description: The ID of the availability zone.
+        required: true
+      description:
+        type: string
+        description: The description of the cluster.
+        required: false
+      account:
+        type: string
+        description: The account associated with the VPC.
+        required: false
+      domain-id:
+        type: string
+        description: The domain ID associated with the VPC.
+        required: false
+      project-id:
+        type: string
+        description: The project ID associated with the VPC.
+        required: false
+      kubernetes-version-id:
+        type: string
+        description: Kubernetes version ID used to launch the cluster.
+        required: false
+      network-id:
+        type: string
+        description: The ID of the network in which the cluster will be launched.
+        required: false
+      node-root-disk-size:
+        type: integer
+        description: Root disk size (GB) for each node.
+        required: false
+      service-offering-id:
+        type: string
+        description: Service offering ID for the VMs in the cluster.
+        required: false
+      size:
+        type: integer
+        description: Number of worker nodes in the cluster.
+        required: false
+      control-nodes:
+        type: integer
+        description: Number of control nodes in the cluster.
+        required: false
+      key-pair:
+        type: string
+        description: SSH key pair name for logging into the VMs.
+        required: false
+      cluster-type:
+        type: string
+        description: Kubernetes cluster type (CloudManaged or ExternalManaged).
+        required: false
+        validation: { $valid_values: [ $value, [CloudManaged, ExternalManaged] ] }
+      docker-registry-password:
+        type: string
+        description: Password for the Docker image private registry.
+        required: false
+      docker-registry-url:
+        type: string
+        description: URL for the Docker image private registry.
+        required: false
+      docker-registry-username:
+        type: string
+        description: Username for the Docker image private registry.
+        required: false
+      external-load-balancer-ip-address:
+        type: string
+        description: External load balancer IP address for HA clusters.
+        required: false
+    attributes:
+      id:
+        type: string
+        description: The ID of the Kubernetes cluster.

engine/schema/src/main/resources/nimble/resource_types/volume.yaml

@@ -0,0 +1,33 @@
+tosca_definitions_version: tosca_2_0
+
+description: >
+  Apache CloudStack TOSCA profile Volume node type definition.
+
+node_types:
+  Volume:
+    description: >
+      Apache CloudStack Volume node type.
+    derived_from: Root
+    attributes:
+      id:
+        type: string
+        description: ID of the volume.
+      name:
+        type: string
+        description: The name of the volume.
+      disk-offering-id:
+        type: string
+        description: The ID of the disk offering used by the volume.
+    properties:
+      zone-id:
+        type: string
+        description: The ID of the zone in which the volume will be created.
+        required: true
+      disk-offering-id:
+        type: string
+        description: The ID of the disk offering for the volume.
+        required: true
+      name:
+        type: string
+        description: The name for the volume.
+        required: true

plugins/iac/nimble/src/main/java/org/apache/cloudstack/persistence/iactemplatesprofile/IacTemplatesProfileDao.java

@@ -18,6 +18,6 @@
 
 import com.cloud.utils.db.GenericDao;
 
-public interface IacTemplatesProfileDao extends GenericDao<IacTemplatesProfile, Long> {
+public interface IacTemplatesProfileDao extends GenericDao<IacTemplatesProfileVO, Long> {
 
 }

plugins/iac/nimble/src/main/java/org/apache/cloudstack/persistence/iactemplatesprofile/IacTemplatesProfileDaoImpl.java

@@ -20,5 +20,5 @@
 import org.springframework.stereotype.Component;
 
 @Component
-public class IacTemplatesProfileDaoImpl extends GenericDaoBase<IacTemplatesProfile, Long> implements IacTemplatesProfileDao {
+public class IacTemplatesProfileDaoImpl extends GenericDaoBase<IacTemplatesProfileVO, Long> implements IacTemplatesProfileDao {
 }

plugins/iac/nimble/src/main/java/org/apache/cloudstack/service/NimbleManagerImpl.java

@@ -19,8 +19,8 @@
 import com.cloud.utils.component.ManagerBase;
 import org.apache.cloudstack.api.command.ListIacResourceTypesCmd;
 import org.apache.cloudstack.framework.config.ConfigKey;
-import org.apache.cloudstack.persistence.iactemplatesprofile.IacTemplatesProfile;
 import org.apache.cloudstack.persistence.iactemplatesprofile.IacTemplatesProfileDao;
+import org.apache.cloudstack.persistence.iactemplatesprofile.IacTemplatesProfileVO;
 import org.apache.cloudstack.tosca.model.ToscaNodeType;
 import org.apache.cloudstack.tosca.parser.ToscaParser;
 
@@ -63,7 +63,7 @@ public boolean configure(String name, Map<String, Object> params) throws Configu
 
     protected Map<String, ToscaNodeType> loadToscaProfile() {
         logger.info("Loading NIMBLE's TOSCA profile.");
-        List<IacTemplatesProfile> profileElements = iacTemplatesProfileDao.listAll();
+        List<IacTemplatesProfileVO> profileElements = iacTemplatesProfileDao.listAll();
 
         return profileElements.stream()
                 .map(element -> toscaParser.parseNodeType(Paths.get(String.format("%s/%s", NIMBLE_PROFILE_FOLDER_PATH, element.getElementContentFilePath()))))

plugins/iac/nimble/src/main/java/org/apache/cloudstack/tosca/model/ToscaAttributeDefinition.java

@@ -4,9 +4,4 @@ public class ToscaAttributeDefinition extends ToscaFieldDefinition {
     public ToscaAttributeDefinition(String name, String description, ToscaPrimitiveType type) {
         super(name, description, type);
     }
-
-    @Override
-    public String toString() {
-        return String.format("%s: %s", this.getClass().getName(), super.toString());
-    }
 }

plugins/iac/nimble/src/main/java/org/apache/cloudstack/tosca/model/ToscaFieldDefinition.java

@@ -27,6 +27,6 @@ public ToscaPrimitiveType getType() {
 
     @Override
     public String toString() {
-        return ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "name", "description", "type");
+        return ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "name", "type");
     }
 }

plugins/iac/nimble/src/main/java/org/apache/cloudstack/tosca/model/ToscaNodeType.java

@@ -29,6 +29,7 @@ public Map<String, ToscaAttributeDefinition> getAttributes() {
 
     @Override
     public String toString() {
-        return ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "name", "properties", "attributes");
+        return String.format("%s: %s", name,
+                ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "properties", "attributes"));
     }
 }