general fixes

Author: bernardodemarco

engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42100to42200.java

@@ -151,7 +151,7 @@ protected void populateNimbleIacResourceTypes(Connection conn) {
     }
 
     private void insertIacResourceType(Connection conn, String fileName, String resourceName, String resourceCategory) {
-        String insertResourceTypeQuery = "INSERT INTO iac_resource_types (uuid, name, category, element_content) VALUES (UUID(), ?, ?, ?)";
+        String insertResourceTypeQuery = "INSERT INTO iac_resource_types (uuid, name, category, content) VALUES (UUID(), ?, ?, ?)";
         String filePath = NIMBLE_RESOURCE_TYPES_DIRECTORY.resolve(fileName).toString();
         logger.debug("Inserting resource type [name: {}] in the database.", filePath);
         try (InputStream inputStream = Thread.currentThread().getContextClassLoader().getResourceAsStream(filePath);

plugins/iac/nimble/src/main/java/org/apache/cloudstack/api/command/DeployIacTemplateCmd.java

@@ -16,13 +16,18 @@
 // under the License.
 package org.apache.cloudstack.api.command;
 
+import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.BaseAsyncCmd;
 import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.response.SuccessResponse;
+import org.apache.cloudstack.persistence.iactemplatesprofile.IacResourceType;
 import org.apache.cloudstack.service.NimbleService;
 
 import javax.inject.Inject;
 
+@APICommand(name = "deployIacTemplate", description = "", responseObject = SuccessResponse.class, entityType = {IacResourceType.class},
+        requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
 public class DeployIacTemplateCmd extends BaseAsyncCmd {
     @Inject
     private NimbleService nimbleService;

plugins/iac/nimble/src/main/java/org/apache/cloudstack/service/NimbleManagerImpl.java

@@ -18,6 +18,7 @@
 
 import com.cloud.utils.Pair;
 import com.cloud.utils.component.ManagerBase;
+import org.apache.cloudstack.api.command.DeployIacTemplateCmd;
 import org.apache.cloudstack.api.command.ListIacResourceTypesCmd;
 import org.apache.cloudstack.api.response.IacResourceTypeResponse;
 import org.apache.cloudstack.api.response.ListResponse;
@@ -85,7 +86,7 @@ public List<Class<?>> getCommands() {
         if (!NimbleServiceEnabled.value()) {
             return commands;
         }
-        return List.of(ListIacResourceTypesCmd.class);
+        return List.of(ListIacResourceTypesCmd.class, DeployIacTemplateCmd.class);
     }
 
     @Override

plugins/iac/nimble/src/main/java/org/apache/cloudstack/service/NimbleService.java

@@ -32,7 +32,7 @@ public interface NimbleService extends PluggableService, Configurable {
             "Number of threads in the global pool used to execute NIMBLE node template provisioning tasks. " +
                     "This pool is initialized during NIMBLE startup using the configured value. Administrators should " +
                     "tune this setting based on service utilization to optimize provisioning performance.",
-            true, NimbleServiceEnabled.key());
+            false, NimbleServiceEnabled.key());
 
     ListResponse<IacResourceTypeResponse> listIacResourceTypes(ListIacResourceTypesCmd cmd);
     void deployIacTemplate(String iacTemplateContent);

plugins/iac/nimble/src/main/java/org/apache/cloudstack/tosca/ToscaOrchestrator.java

@@ -26,10 +26,10 @@
 import org.apache.logging.log4j.Logger;
 
 import javax.inject.Inject;
-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -52,53 +52,41 @@ public class ToscaOrchestrator {
     public void deployIacTemplate(String iacTemplateContent) {
         logger.debug("Parsing service template");
         ToscaServiceTemplate serviceTemplate = toscaParser.parseServiceTemplate(iacTemplateContent, toscaProfile, null);
-        Map<ToscaNodeTemplate, CompletableFuture<String>> provisioningTasksFutures = createProvisioningTasksFutures(serviceTemplate);
+        Map<String, CompletableFuture<String>> provisioningTasksFutures = createProvisioningTasksFutures(serviceTemplate);
         CompletableFuture<Void> serviceTemplateFeature = CompletableFuture.allOf(provisioningTasksFutures.values().toArray(new CompletableFuture[0]));
         serviceTemplateFeature.join();
     }
 
-    private Map<ToscaNodeTemplate, CompletableFuture<String>> createProvisioningTasksFutures(ToscaServiceTemplate serviceTemplate) {
+    private Map<String, CompletableFuture<String>> createProvisioningTasksFutures(ToscaServiceTemplate serviceTemplate) {
         logger.debug("Creating provisioning tasks futures");
-        Map<ToscaNodeTemplate, CompletableFuture<String>> futures = new HashMap<>();
-        Map<String, Set<ToscaNodeTemplate>> dependencyGraph = serviceTemplate.getDependencyGraph();
-
-        List<String> nodesWithoutDependencies = dependencyGraph.keySet().stream().filter(node -> dependencyGraph.get(node).isEmpty()).collect(Collectors.toList());
-        logger.debug("Nodes without dependencies: " + nodesWithoutDependencies);
-        nodesWithoutDependencies.forEach(node -> {
+        Map<String, CompletableFuture<String>> futures = new HashMap<>();
+        getServiceTemplateTopologicalSort(serviceTemplate).forEach((node, dependencies) -> {
             ToscaNodeTemplate nodeTemplate = serviceTemplate.getNodeTemplates().get(node);
-            CompletableFuture<String> taskFuture = buildNodeProvisioningTask(nodeTemplate);
-            futures.put(nodeTemplate, taskFuture);
-        });
-        logger.debug("Built all the features for the nodes without dependencies.");
-
-        logger.debug("Service template size: " + serviceTemplate.getNodeTemplates().size());
-        while (futures.size() < serviceTemplate.getNodeTemplates().size()) {
-            logger.debug("Futures size: " + futures.size());
-            List<ToscaNodeTemplate> nodesWhoseDependenciesAlreadyHaveFutures = dependencyGraph.keySet().stream()
-                    .filter(node -> !dependencyGraph.get(node).isEmpty() && dependencyGraph.get(node).stream().allMatch(futures::containsKey))
-                    .map(node -> serviceTemplate.getNodeTemplates().get(node))
-                    .collect(Collectors.toList());
-
-            for (ToscaNodeTemplate node : nodesWhoseDependenciesAlreadyHaveFutures) {
-                CompletableFuture<?>[] dependencies = dependencyGraph.get(node.getName()).stream().map(futures::get).toArray(CompletableFuture[]::new);
-                CompletableFuture<String> taskFuture = CompletableFuture.allOf(dependencies).thenCompose(v -> {
-                    logger.debug("All dependencies of the node [" + node.getName() + "] are ready.");
+            CompletableFuture<String> taskFuture;
+            if (dependencies.isEmpty()) {
+                logger.debug("Node [{}] has no dependencies. Building its provisioning task.", node);
+                taskFuture = buildNodeProvisioningTask(nodeTemplate);
+            } else {
+                CompletableFuture<?>[] dependenciesFutures = dependencies.stream()
+                        .map((dep) -> futures.get(dep.getName())).toArray(CompletableFuture[]::new);
+                taskFuture = CompletableFuture.allOf(dependenciesFutures).thenCompose(v -> {
+                    logger.debug("All dependencies of the node [{}] are ready.", node);
                     logger.debug("Here you'll be able to resolve the unresolved properties by get property and get attribute");
-                    return buildNodeProvisioningTask(node);
+                    return buildNodeProvisioningTask(nodeTemplate);
                 });
-
-                futures.put(node, taskFuture);
             }
-        }
+
+            futures.put(node, taskFuture);
+        });
 
         return futures;
     }
 
     // O(|V|+|E|)
-    private List<String> getServiceTemplateTopologicalSort(ToscaServiceTemplate serviceTemplate) {
+    private LinkedHashMap<String, Set<ToscaNodeTemplate>> getServiceTemplateTopologicalSort(ToscaServiceTemplate serviceTemplate) {
+        LinkedHashMap<String, Set<ToscaNodeTemplate>> topologicalSort = new LinkedHashMap<>();
         Set<String> visitedNodes = new HashSet<>();
         Set<String> branchAncestors = new HashSet<>();
-        List<String> topologicalSort = new ArrayList<>();
         for (ToscaNodeTemplate node : serviceTemplate.getNodeTemplates().values()) {
             if (!visitedNodes.contains(node.getName())) {
                 depthFirstSearch(topologicalSort, node.getName(), serviceTemplate.getDependencyGraph(), branchAncestors, visitedNodes);
@@ -107,7 +95,7 @@ private List<String> getServiceTemplateTopologicalSort(ToscaServiceTemplate serv
         return topologicalSort;
     }
 
-    private void depthFirstSearch(List<String> topologicalSort, String node, Map<String, Set<ToscaNodeTemplate>> graph, Set<String> branchAncestors, Set<String> visitedNodes) {
+    private void depthFirstSearch(LinkedHashMap<String, Set<ToscaNodeTemplate>> topologicalSort, String node, Map<String, Set<ToscaNodeTemplate>> graph, Set<String> branchAncestors, Set<String> visitedNodes) {
         visitedNodes.add(node);
         branchAncestors.add(node);
         for (ToscaNodeTemplate dependency : graph.getOrDefault(node, Collections.emptySet())) {
@@ -121,7 +109,7 @@ private void depthFirstSearch(List<String> topologicalSort, String node, Map<Str
         }
 
         branchAncestors.remove(node);
-        topologicalSort.add(node);
+        topologicalSort.put(node, graph.getOrDefault(node, Collections.emptySet()));
     }
 
     private CompletableFuture<String> buildNodeProvisioningTask(ToscaNodeTemplate nodeTemplate) {

plugins/iac/nimble/src/main/java/org/apache/cloudstack/tosca/model/ToscaProperty.java

@@ -19,7 +19,7 @@
 public class ToscaProperty {
     private final ToscaPropertyDefinition definition;
     private final Object rawValue;
-    private final Object evaluatedValue;
+    private volatile Object evaluatedValue;
 
     public ToscaProperty(ToscaPropertyDefinition definition, Object rawValue, Object evaluatedValue) {
         this.definition = definition;
@@ -35,6 +35,10 @@ public Object getEvaluatedValue() {
         return evaluatedValue;
     }
 
+    public void setEvaluatedValue(Object evaluatedValue) {
+        this.evaluatedValue = evaluatedValue;
+    }
+
     public ToscaPropertyDefinition getDefinition() {
         return definition;
     }

plugins/iac/nimble/src/main/java/org/apache/cloudstack/tosca/model/ToscaServiceTemplate.java

@@ -17,7 +17,6 @@
 package org.apache.cloudstack.tosca.model;
 
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
 

plugins/iac/nimble/src/main/java/org/apache/cloudstack/tosca/parser/ToscaServiceTemplateParser.java

@@ -67,7 +67,7 @@ public ToscaServiceTemplate parseServiceTemplate(String content, Map<String, Tos
             throw new InvalidParameterValueException(context.buildErrorMessages());
         }
 
-        logger.info("The TOSCA service template has been successfully parsed and validated..");
+        logger.info("The TOSCA service template has been successfully parsed and validated.");
         return new ToscaServiceTemplate(nodeTemplates, dependencyGraph, inputs, context.getUnresolvedByGetInput());
     }
 
@@ -441,7 +441,7 @@ private Map<String, Set<ToscaNodeTemplate>> buildDependencyGraph(Map<String, Tos
             });
         }
 
-        logger.info("Dependency graph of the TOSCA service template has been successfully built with [{}] nodes and [{}] arcs/dependencies.", graph.keySet(), graph.values().stream().mapToLong(Set::size).sum());
+        logger.info("Dependency graph of the TOSCA service template has been successfully built with [{}] arcs/dependencies.", graph.values().stream().mapToLong(Set::size).sum());
         return graph;
     }