Generate parameters per each resource file (#784)

* Generate parameters per each resource file
* Add parameter for redefining parameters output folder
* Update documentation with new parameter

Co-authored-by: Farhad Alizada <falizada@microsoft.com>

Author: f-alizada

src/ArmTemplates/Commands/Configurations/ExtractorConsoleAppConfiguration.cs

@@ -106,6 +106,9 @@ public class ExtractorConsoleAppConfiguration
         [Option(longName: "extractIdentityProviders", HelpText = "Extract identity providers from the service if applies")]
         public string ExtractIdentityProviders { get; set; }
 
+        [Option(longName: "parametersOutputDirectoryName ", HelpText = "Parameters output directory name, by default it is \"parameters\"")]
+        public string ParametersOutputDirectoryName { get; set; }
+
         /// <summary>
         /// Api parameter properties for overriding Api OAuth2 scope or/and Service urloverride. Available via extractor-config file only.
         /// </summary>

src/ArmTemplates/Commands/Executors/ExtractorExecutor.cs

@@ -410,6 +410,7 @@ public async Task<Template<ApiTemplateResources>> GenerateApiTemplateAsync(
             if (apiTemplate?.HasResources() == true)
             {
                 apiTemplate.TypedResources.FileName = FileNameGenerator.GenerateExtractorAPIFileName(singleApiName, this.extractorParameters.FileNames.BaseFileName);
+                apiTemplate.TypedResources.ParametersFileName = FileNameGenerator.GenerateExtractorAPIParametersFileName(singleApiName);
 
                 await FileWriter.SaveAsJsonAsync(
                     apiTemplate,
@@ -459,6 +460,79 @@ await FileWriter.SaveAsJsonAsync(
             return templateParameters;
         }
 
+        public async Task GenerateResourceParametersFiles(
+            string baseFilesGenerationDirectory,
+            Template mainParametersTemplate,
+            Template<ApiTemplateResources> apiTemplate = null,
+            Template<PolicyTemplateResources> policyTemplate = null,
+            Template<ApiVersionSetTemplateResources> apiVersionSetTemplate = null,
+            Template<ProductTemplateResources> productsTemplate = null,
+            Template<ProductApiTemplateResources> productApisTemplate = null,
+            Template<TagApiTemplateResources> apiTagsTemplate = null,
+            Template<LoggerTemplateResources> loggersTemplate = null,
+            Template<BackendTemplateResources> backendsTemplate = null,
+            Template<AuthorizationServerTemplateResources> authorizationServersTemplate = null,
+            Template<NamedValuesResources> namedValuesTemplate = null,
+            Template<TagTemplateResources> tagTemplate = null,
+            Template<GroupTemplateResources> groupTemplate = null,
+            Template<IdentityProviderResources> identityProviderTemplate = null,
+            Template<SchemaTemplateResources> schemaTemplate = null,
+            Template<OpenIdConnectProviderResources> openIdConnectProviderTemplate = null,
+            Template<PolicyFragmentsResources> policyFragmentsTemplate = null)
+        {
+            this.RenameExistingParametersDirectory(baseFilesGenerationDirectory);
+
+            await this.GenerateResourceParametersFile(baseFilesGenerationDirectory, apiTemplate?.TypedResources.ParametersFileName, apiTemplate, mainParametersTemplate);
+            await this.GenerateResourceParametersFile(baseFilesGenerationDirectory, this.extractorParameters.FileNames.GlobalServicePolicyParameters , policyTemplate, mainParametersTemplate);
+            await this.GenerateResourceParametersFile(baseFilesGenerationDirectory, this.extractorParameters.FileNames.ApiVersionSetsParameters, apiVersionSetTemplate, mainParametersTemplate);
+            await this.GenerateResourceParametersFile(baseFilesGenerationDirectory, this.extractorParameters.FileNames.ProductsParameters, productsTemplate, mainParametersTemplate);
+            await this.GenerateResourceParametersFile(baseFilesGenerationDirectory, this.extractorParameters.FileNames.ProductAPIsParameters, productApisTemplate, mainParametersTemplate);
+            await this.GenerateResourceParametersFile(baseFilesGenerationDirectory, this.extractorParameters.FileNames.TagApiParameters, apiTagsTemplate, mainParametersTemplate);
+            await this.GenerateResourceParametersFile(baseFilesGenerationDirectory, this.extractorParameters.FileNames.LoggersParameters, loggersTemplate, mainParametersTemplate);
+            await this.GenerateResourceParametersFile(baseFilesGenerationDirectory, this.extractorParameters.FileNames.BackendsParameters, backendsTemplate, mainParametersTemplate);
+            await this.GenerateResourceParametersFile(baseFilesGenerationDirectory, this.extractorParameters.FileNames.AuthorizationServersParameters, authorizationServersTemplate, mainParametersTemplate);
+            await this.GenerateResourceParametersFile(baseFilesGenerationDirectory, this.extractorParameters.FileNames.NamedValuesParameters, namedValuesTemplate, mainParametersTemplate);
+            await this.GenerateResourceParametersFile(baseFilesGenerationDirectory, this.extractorParameters.FileNames.TagsParameters, tagTemplate, mainParametersTemplate);
+            await this.GenerateResourceParametersFile(baseFilesGenerationDirectory, this.extractorParameters.FileNames.GroupsParameters, groupTemplate, mainParametersTemplate);
+            await this.GenerateResourceParametersFile(baseFilesGenerationDirectory, this.extractorParameters.FileNames.IdentityProvidersParameters, identityProviderTemplate, mainParametersTemplate);
+            await this.GenerateResourceParametersFile(baseFilesGenerationDirectory, this.extractorParameters.FileNames.SchemaParameters, schemaTemplate, mainParametersTemplate);
+            await this.GenerateResourceParametersFile(baseFilesGenerationDirectory, this.extractorParameters.FileNames.OpenIdConnectProvidersParameters, openIdConnectProviderTemplate, mainParametersTemplate);
+            await this.GenerateResourceParametersFile(baseFilesGenerationDirectory, this.extractorParameters.FileNames.PolicyFragmentsParameters, policyFragmentsTemplate, mainParametersTemplate);
+        }
+
+        public async Task GenerateResourceParametersFile<TTemplateResource>(string baseFilesGenerationDirectory, string fileName, Template<TTemplateResource> resourceTemplate, Template mainParametersTemplate) where TTemplateResource : ITemplateResources, new()
+        {
+            if (string.IsNullOrEmpty(fileName) || string.IsNullOrEmpty(baseFilesGenerationDirectory))
+            {
+                this.logger.LogWarning("Filename or directory for parameters file generation is null or empty, skipping the generation");
+                return;
+            }
+
+            if (resourceTemplate?.TypedResources?.HasContent() == true)
+            {
+                var parametersTemplate = this.parametersExtractor.CreateResourceTemplateParameterTemplate(resourceTemplate, mainParametersTemplate);
+
+                if (!parametersTemplate.Parameters.IsNullOrEmpty())
+                {
+                    await FileWriter.SaveAsJsonAsync(
+                    parametersTemplate,
+                    directory: Path.Combine(baseFilesGenerationDirectory, this.extractorParameters.FileNames.ParametersDirectory),
+                    fileName: fileName);
+                }
+            }
+        }
+
+        void RenameExistingParametersDirectory(string baseDirectoryLocation)
+        {
+            var parametersLocation = Path.Combine(baseDirectoryLocation, this.extractorParameters.FileNames.ParametersDirectory);
+            if (Directory.Exists(parametersLocation))
+            {
+                var creationDateTime = Directory.GetCreationTime(parametersLocation).ToString("yyyyMMddHHmmss");
+                var newParameterDirectory = $"{this.extractorParameters.FileNames.ParametersDirectory}{creationDateTime}";
+                Directory.Move(parametersLocation, Path.Combine(baseDirectoryLocation, newParameterDirectory));
+            }
+        }
+
         public async Task<Template<MasterTemplateResources>> GenerateMasterTemplateAsync(
            string baseFilesGenerationDirectory,
            ApiTemplateResources apiTemplateResources = null,
@@ -1035,8 +1109,28 @@ async Task GenerateTemplates(
             await this.GenerateGatewayTemplateAsync(singleApiName, baseFilesGenerationDirectory);
             await this.GenerateGatewayApiTemplateAsync(singleApiName, multipleApiNames, baseFilesGenerationDirectory);
             await this.GenerateApiManagementServiceTemplate(baseFilesGenerationDirectory);
-            await this.GenerateParametersTemplateAsync(apisToExtract, loggerTemplate.TypedResources, backendTemplate.TypedResources, namedValueTemplate.TypedResources, identityProviderTemplate.TypedResources, openIdConnectProviderTemplate.TypedResources, baseFilesGenerationDirectory);
+            var parametersTemplate = await this.GenerateParametersTemplateAsync(apisToExtract, loggerTemplate.TypedResources, backendTemplate.TypedResources, namedValueTemplate.TypedResources, identityProviderTemplate.TypedResources, openIdConnectProviderTemplate.TypedResources, baseFilesGenerationDirectory);
 
+            await this.GenerateResourceParametersFiles(
+                baseFilesGenerationDirectory, 
+                parametersTemplate, 
+                apiTemplate: apiTemplate,
+                policyTemplate: globalServicePolicyTemplate,
+                productApisTemplate: productApiTemplate,
+                productsTemplate: productTemplate,
+                apiVersionSetTemplate: apiVersionSetTemplate,
+                authorizationServersTemplate: authorizationServerTemplate,
+                tagTemplate: tagTemplate,
+                apiTagsTemplate: apiTagTemplate,
+                loggersTemplate: loggerTemplate,
+                namedValuesTemplate: namedValueTemplate,
+                backendsTemplate: backendTemplate,
+                groupTemplate: groupTemplate,
+                identityProviderTemplate: identityProviderTemplate,
+                openIdConnectProviderTemplate: openIdConnectProviderTemplate,
+                schemaTemplate: schemasTempate,
+                policyFragmentsTemplate: policyFragmentTemplate);
+
             await this.GenerateMasterTemplateAsync(
                 baseFilesGenerationDirectory,
                 apiTemplateResources: apiTemplate.TypedResources,

src/ArmTemplates/Common/FileHandlers/FileNameGenerator.cs

@@ -24,22 +24,38 @@ public static FileNames GenerateFileNames(string baseFileName)
             return new FileNames
             {
                 ApiVersionSets = $@"{baseFileName}apiVersionSets.template.json",
+                ApiVersionSetsParameters = "apiVersionSets.parameters.json",
                 AuthorizationServers = $@"{baseFileName}authorizationServers.template.json",
+                AuthorizationServersParameters = "authorizationServers.parameters.json",
                 Backends = $@"{baseFileName}backends.template.json",
+                BackendsParameters = "backends.parameters.json",
                 GlobalServicePolicy = $@"{baseFileName}globalServicePolicy.template.json",
+                GlobalServicePolicyParameters = "globalServicePolicy.parameters.json",
                 Loggers = $@"{baseFileName}loggers.template.json",
+                LoggersParameters = "loggers.parameters.json",
                 NamedValues = $@"{baseFileName}namedValues.template.json",
+                NamedValuesParameters = "namedValues.parameters.json",
                 Tags = $@"{baseFileName}tags.template.json",
+                TagsParameters = "tags.parameters.json",
                 Products = $@"{baseFileName}products.template.json",
+                ProductsParameters = "products.parameters.json",
                 ProductAPIs = $@"{baseFileName}productAPIs.template.json",
+                ProductAPIsParameters = "productAPIs.parameters.json",
                 Gateway = $@"{baseFileName}gateways.template.json",
+                GatewayParameters = "gateways.parameters.json",
                 GatewayApi = $@"{baseFileName}gateways-apis.template.json",
+                GatewayApiParameters = "gateways-apis.parameters.json",
                 IdentityProviders = $@"{baseFileName}identity-providers.template.json",
+                IdentityProvidersParameters = "identity-providers.parameters.json",
                 OpenIdConnectProviders = $@"{baseFileName}openid-connect-providers.template.json",
+                OpenIdConnectProvidersParameters = "openid-connect-providers.parameters.json",
                 ApiManagementService = $@"{baseFileName}api-management-service.template.json",
                 TagApi = $@"{baseFileName}apiTags.template.json",
+                TagApiParameters = "apiTags.parameters.json",
                 Schema = $@"{baseFileName}schemas.template.json",
+                SchemaParameters = "schemas.parameters.json",
                 PolicyFragments = $@"{baseFileName}policy-fragments.template.json",
+                PolicyFragmentsParameters = "policy-fragments.parameters.json",
                 Parameters = $@"{baseFileName}parameters.json",
                 LinkedMaster = $@"{baseFileName}master.template.json",
                 Apis = "/Apis",
@@ -48,6 +64,8 @@ public static FileNames GenerateFileNames(string baseFileName)
                 RevisionMasterFolder = "/RevisionMasterFolder",
                 GroupAPIsMasterFolder = "/MultipleApisMasterFolder",
                 Groups = $"{baseFileName}groups.template.json",
+                GroupsParameters = "groups.parameters.json",
+                ParametersDirectory = $"{baseFileName}parameters",
                 BaseFileName = baseFileName
             };
         }
@@ -69,7 +87,16 @@ public static string GenerateCreatorAPIFileName(string apiName, bool isSplitAPI,
 
         public static string GenerateExtractorAPIFileName(string singleAPIName, string baseFileName)
         {
-            return singleAPIName == null ? $@"{baseFileName}apis.template.json" : $@"{baseFileName}{singleAPIName}-api.template.json";
+            return $@"{baseFileName}{GenerateApiFileNameBase(singleAPIName)}.template.json";
+        }
+
+        public static string GenerateExtractorAPIParametersFileName(string singleAPIName)
+        {
+            return $@"{GenerateApiFileNameBase(singleAPIName)}.parameters.json";
+        }
+
+        public static string GenerateApiFileNameBase(string singleAPIName) {
+            return singleAPIName == null ? "apis" : $"{singleAPIName}-api";
         }
 
         public static string GenerateOriginalAPIName(string apiName)

src/ArmTemplates/Common/FileHandlers/FileNames.cs

@@ -9,40 +9,74 @@ public class FileNames
     {
         public string ApiVersionSets { get; set; }
 
+        public string ApiVersionSetsParameters { get; set; }
+
         public string AuthorizationServers { get; set; }
 
+        public string AuthorizationServersParameters { get; set; }
+
         public string Backends { get; set; }
 
+        public string BackendsParameters { get; set; }
+
         public string GlobalServicePolicy { get; set; }
 
+        public string GlobalServicePolicyParameters { get; set; }
+
         public string Loggers { get; set; }
 
+        public string LoggersParameters { get; set; }
+
         public string NamedValues { get; set; }
 
+        public string NamedValuesParameters { get; set; }
+
         public string Tags { get; set; }
 
+        public string TagsParameters { get; set; }
+
         public string Products { get; set; }
 
+        public string ProductsParameters { get; set; }
+
         public string ProductAPIs { get; set; }
 
+        public string ProductAPIsParameters { get; set; }
+
         public string Groups { get; set; }
 
+        public string GroupsParameters { get; set; }
+
         public string TagApi { get; set; }
 
+        public string TagApiParameters { get; set; }
+
         public string Gateway { get; set; }
 
+        public string GatewayParameters { get; set; }
+
         public string GatewayApi { get; set; }
 
+        public string GatewayApiParameters { get; set; }
+
         public string IdentityProviders { get; set; }
 
+        public string IdentityProvidersParameters { get; set; }
+
         public string OpenIdConnectProviders { get; set; }
 
+        public string OpenIdConnectProvidersParameters { get; set; }
+
         public string ApiManagementService { get; set; }
 
         public string Schema { get; set; }
 
+        public string SchemaParameters { get; set; }
+
         public string PolicyFragments { get; set; }
 
+        public string PolicyFragmentsParameters { get; set; }
+
         public string Parameters { get; set; }
 
         // linked property outputs 1 master template
@@ -59,5 +93,7 @@ public class FileNames
         public string GroupAPIsMasterFolder { get; set; }
 
         public string BaseFileName { get; set; }
+
+        public string ParametersDirectory { get; set; }
     }
 }

src/ArmTemplates/Common/Templates/Apis/ApiTemplateResources.cs

@@ -19,6 +19,8 @@ public class ApiTemplateResources : TemplateResourcesBase, ITemplateResources
     {
         public string FileName { get; set; }
 
+        public string ParametersFileName { get; set; }
+
         public List<ApiTemplateResource> Apis { get; set; } = new();
 
         public List<ApiSchemaTemplateResource> ApiSchemas { get; set; } = new();

src/ArmTemplates/Extractor/EntityExtractors/Abstractions/IParametersExtractor.cs

@@ -27,5 +27,7 @@ Task<Template> CreateMasterTemplateParameterValues(
             IdentityProviderResources identityProviderResources,
             OpenIdConnectProviderResources openIdConnectProviderResources,
             ExtractorParameters extractorParameters);
+
+        Template CreateResourceTemplateParameterTemplate(Template resourceTemplate, Template mainParametersTemplate);
     }
 }

src/ArmTemplates/Extractor/EntityExtractors/ParametersExtractor.cs

@@ -18,6 +18,7 @@
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.OpenIdConnectProviders;
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Extractor.EntityExtractors.Abstractions;
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Extractor.Models;
+using Microsoft.Extensions.Logging;
 
 namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Extractor.EntityExtractors
 {
@@ -27,13 +28,16 @@ public class ParametersExtractor : IParametersExtractor
         readonly IApisClient apisClient;
         readonly IIdentityProviderClient identityProviderClient;
         readonly IOpenIdConnectProvidersClient openIdConnectProviderClient;
+        readonly ILogger<ParametersExtractor> logger;
 
         public ParametersExtractor(
+            ILogger<ParametersExtractor> logger,
             ITemplateBuilder templateBuilder,
             IApisClient apisClient,
             IIdentityProviderClient identityProviderClient,
             IOpenIdConnectProvidersClient openIdConnectProviderClient)
         {
+            this.logger = logger;
             this.templateBuilder = templateBuilder;
             this.apisClient = apisClient;
             this.identityProviderClient = identityProviderClient;
@@ -268,5 +272,31 @@ async Task AddSecretValuesParameters()
 
             return parametersTemplate;
         }
+
+        public Template CreateResourceTemplateParameterTemplate(Template resourceTemplate, Template mainParameterTemplate)
+        {
+            var parametersTemplate = this.templateBuilder
+                .GenerateEmptyTemplate()
+                .Build();
+            parametersTemplate.Parameters = new();
+            var parameters = parametersTemplate.Parameters;
+
+            if (resourceTemplate?.Parameters.IsNullOrEmpty() != true)
+            {
+                foreach (var parameterKey in resourceTemplate.Parameters.Keys)
+                {
+                    if (!mainParameterTemplate.Parameters.ContainsKey(parameterKey)) 
+                    {
+                        this.logger.LogWarning($"Parameter {parameterKey} were not found in main parameters template");
+                        continue;
+                    }
+                    
+                    var parameterValue = mainParameterTemplate.Parameters[parameterKey];
+                    parameters.Add(parameterKey, parameterValue);
+                }
+            }
+            
+            return parametersTemplate;
+        }
     }
 }