feat: [CDS-37215]: fix Stage individual schema generation in pipleine service (#34840)

* [CDS-37215]: fix Stage individual schema generation in pipleine service

* [CDS-37215]: snow approavl step entity should belong to PMS

* [CDS-37215]: Add a hack to prevent failure for step schema generation on pipeline side

* [CDS-37215]: fix code format error

Author: abhinavmittal-harness

800-pipeline-service/src/main/java/io/harness/pms/pipeline/service/PMSYamlSchemaServiceImpl.java

@@ -23,6 +23,7 @@
 import io.harness.annotations.dev.HarnessTeam;
 import io.harness.annotations.dev.OwnedBy;
 import io.harness.beans.FeatureName;
+import io.harness.data.structure.EmptyPredicate;
 import io.harness.encryption.Scope;
 import io.harness.exception.InvalidYamlException;
 import io.harness.exception.JsonSchemaException;
@@ -50,16 +51,23 @@
 import io.harness.yaml.utils.YamlSchemaUtils;
 import io.harness.yaml.validator.YamlSchemaValidator;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.inject.Inject;
 import com.google.inject.name.Named;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import java.util.concurrent.Executor;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
@@ -305,18 +313,92 @@ public JsonNode getIndividualYamlSchema(String accountId, String orgIdentifier,
       return getPipelineYamlSchemaInternal(accountId, projectIdentifier, orgIdentifier, null);
     }
     List<YamlSchemaWithDetails> yamlSchemaWithDetailsList = null;
+    Map<String, List<JsonNode>> nameSpaceToDefinitionMap = new HashMap<>();
+    Set<String> nameSpaces = new HashSet<>();
+    JsonNode mergedDefinition = null;
+    Map<String, JsonNode> finalNameSpaceToDefinitionMap = new HashMap<>();
     if (StepCategory.STAGE.toString().equals(yamlGroup)) {
       List<ModuleType> enabledModules = obtainEnabledModules(accountId);
+      enabledModules.add(ModuleType.PMS);
       yamlSchemaWithDetailsList = fetchSchemaWithDetailsFromModules(accountId, enabledModules);
+      yamlSchemaWithDetailsList =
+          filterYamlSchemaDetailsByModule(yamlSchemaWithDetailsList, entityType.getEntityProduct());
+      for (YamlSchemaWithDetails yamlSchemaWithDetails : yamlSchemaWithDetailsList) {
+        String nameSpace = yamlSchemaWithDetails.getYamlSchemaMetadata().getNamespace();
+        JsonNode definition = yamlSchemaWithDetails.getSchema().get(DEFINITIONS_NODE);
+        nameSpaces.add(nameSpace);
+        if (EmptyPredicate.isEmpty(nameSpace)) {
+          if (mergedDefinition == null) {
+            mergedDefinition = definition;
+          } else {
+            JsonNodeUtils.merge(mergedDefinition, definition);
+          }
+        } else {
+          if (nameSpaceToDefinitionMap.containsKey(nameSpace)) {
+            nameSpaceToDefinitionMap.get(nameSpace).add(definition);
+          } else {
+            List<JsonNode> nameSpaceDefinition = new LinkedList<>();
+            nameSpaceDefinition.add(definition);
+            nameSpaceToDefinitionMap.put(nameSpace, nameSpaceDefinition);
+          }
+        }
+      }
+      for (Map.Entry<String, List<JsonNode>> entry : nameSpaceToDefinitionMap.entrySet()) {
+        JsonNode nameSpaceDefinition = null;
+        for (JsonNode jsonNode : entry.getValue()) {
+          if (nameSpaceDefinition == null) {
+            nameSpaceDefinition = jsonNode;
+          } else {
+            JsonNodeUtils.merge(nameSpaceDefinition, jsonNode);
+          }
+        }
+        finalNameSpaceToDefinitionMap.put(entry.getKey(), nameSpaceDefinition);
+      }
     }
     JsonNode jsonNode = schemaFetcher.fetchStepYamlSchema(
         accountId, projectIdentifier, orgIdentifier, scope, entityType, yamlGroup, yamlSchemaWithDetailsList);
 
     // TODO: hack to remove v2 steps from stage yamls. Fix it properly
     if (StepCategory.STAGE.toString().equals(yamlGroup)) {
-      YamlSchemaTransientHelper.removeV2StepEnumsFromStepElementConfig(
-          jsonNode.get(DEFINITIONS_NODE).fields().next().getValue().get(STEP_ELEMENT_CONFIG));
+      String stepNameSpace = null;
+      if (jsonNode.get(DEFINITIONS_NODE).fields().hasNext()) {
+        String nameSpace = jsonNode.get(DEFINITIONS_NODE).fields().next().getKey();
+        if (nameSpaces.contains(nameSpace)) {
+          stepNameSpace = nameSpace;
+        }
+      }
+
+      JsonNodeUtils.merge(jsonNode.get(DEFINITIONS_NODE), mergedDefinition);
+      for (Map.Entry<String, JsonNode> entry : finalNameSpaceToDefinitionMap.entrySet()) {
+        if (!stepNameSpace.equals(entry.getKey())) {
+          JsonNodeUtils.merge(jsonNode.get(DEFINITIONS_NODE), entry.getValue());
+        }
+      }
+      for (String nameSpace : nameSpaces) {
+        if (jsonNode.get(DEFINITIONS_NODE).get(nameSpace) != null) {
+          YamlSchemaTransientHelper.removeV2StepEnumsFromStepElementConfig(
+              jsonNode.get(DEFINITIONS_NODE).get(nameSpace).get(STEP_ELEMENT_CONFIG));
+        }
+      }
+    } else {
+      JsonNode stepSpecTypeNode = getStepSpecType();
+      JsonNodeUtils.merge(jsonNode.get(DEFINITIONS_NODE), stepSpecTypeNode);
     }
     return jsonNode;
   }
+
+  // TODO: Brijesh to look at the intermittent issue and remove this
+  private JsonNode getStepSpecType() {
+    String stepSpecTypeNodeString = "{\"StepSpecType\": {\n"
+        + "                    \"type\": \"object\",\n"
+        + "                    \"discriminator\": \"type\",\n"
+        + "                    \"$schema\": \"http://json-schema.org/draft-07/schema#\"\n"
+        + "                }}";
+    ObjectMapper mapper = new ObjectMapper();
+    try {
+      return mapper.readTree(stepSpecTypeNodeString);
+    } catch (JsonProcessingException e) {
+      throw new RuntimeException(e);
+    }
+  }
 }

840-template-service/src/main/java/io/harness/template/services/NGTemplateSchemaServiceImpl.java

@@ -137,7 +137,10 @@ public void validateYamlSchemaInternal(TemplateEntity templateEntity) {
       return;
     }
     try {
-      Scope scope = projectIdentifier != null ? Scope.PROJECT : orgIdentifier != null ? Scope.ORG : Scope.ACCOUNT;
+      Scope scope = templateEntity.getTemplateScope() != null ? templateEntity.getTemplateScope()
+          : projectIdentifier != null                         ? Scope.PROJECT
+          : orgIdentifier != null                             ? Scope.ORG
+                                                              : Scope.ACCOUNT;
       JsonNode schema = getTemplateSchema(accountIdentifier, projectIdentifier, orgIdentifier, scope,
           templateEntity.getChildType(), templateEntity.getTemplateEntityType());
       String schemaString = JsonPipelineUtils.writeJsonString(schema);

970-ng-commons/src/main/java/io/harness/EntityType.java

@@ -187,7 +187,7 @@ public enum EntityType {
   FEATURE_FLAGS(
       ModuleType.CF, EntityTypeConstants.FEATURE_FLAGS, IdentifierRef.class, EntityYamlRootNames.FEATURE_FLAGS),
   @JsonProperty(EntityTypeConstants.SERVICENOW_APPROVAL)
-  SERVICENOW_APPROVAL_STEP(ModuleType.CD, EntityTypeConstants.SERVICENOW_APPROVAL, IdentifierRef.class,
+  SERVICENOW_APPROVAL_STEP(ModuleType.PMS, EntityTypeConstants.SERVICENOW_APPROVAL, IdentifierRef.class,
       EntityYamlRootNames.SERVICENOW_APPROVAL),
   @JsonProperty(EntityTypeConstants.SERVICENOW_CREATE)
   SERVICENOW_CREATE_STEP(ModuleType.PMS, EntityTypeConstants.SERVICENOW_CREATE, IdentifierRef.class,