Restructure the API template generation, ordering revisions depends on property (#782)

* Add release template, when revisions generated for single api
* Add revision to api name, if not exist. Introduce ApisResource id naming helper. 
* Add local isCurrent property to api template
* Add api releasese file with release resource for each api
* Replace explicit dependson parameter for apis in all api related extractors
* Add tests for release extractor and api data processor
* Add apiRelease parameters file

Co-authored-by: Farhad Alizada <falizada@microsoft.com>

Author: f-alizada

src/ArmTemplates/Commands/Executors/ExtractorExecutor.cs

@@ -11,6 +11,7 @@
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.FileHandlers;
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.Abstractions;
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.ApiManagementService;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.ApiReleases;
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.Apis;
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.ApiVersionSet;
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.AuthorizationServer;
@@ -70,7 +71,8 @@ public class ExtractorExecutor
         readonly ISchemaExtractor schemaExtractor;
         readonly IOpenIdConnectProviderExtractor openIdConnectProviderExtractor;
         readonly IPolicyFragmentsExtractor policyFragmentsExtractor;
-
+        readonly IApiReleaseExtractor apiReleaseExtractor;
+ 
         public ExtractorExecutor(
             ILogger<ExtractorExecutor> logger,
             IApisClient apisClient,
@@ -95,7 +97,8 @@ public ExtractorExecutor(
             IApiManagementServiceExtractor apiManagementServiceExtractor,
             ISchemaExtractor schemaExtractor,
             IOpenIdConnectProviderExtractor openIdConnectProviderExtractor,
-            IPolicyFragmentsExtractor policyFragmentsExtractor)
+            IPolicyFragmentsExtractor policyFragmentsExtractor,
+            IApiReleaseExtractor apiReleaseExtractor)
         {
             this.logger = logger;
             this.apisClient = apisClient;
@@ -121,6 +124,7 @@ public ExtractorExecutor(
             this.schemaExtractor = schemaExtractor;
             this.openIdConnectProviderExtractor = openIdConnectProviderExtractor;
             this.policyFragmentsExtractor = policyFragmentsExtractor;
+            this.apiReleaseExtractor = apiReleaseExtractor;
         }
 
         /// <summary>
@@ -151,7 +155,8 @@ public static ExtractorExecutor BuildExtractorExecutor(
             IApiManagementServiceExtractor apiManagementServiceExtractor = null,
             ISchemaExtractor schemaExtractor = null,
             IOpenIdConnectProviderExtractor openIdConnectProviderExtractor = null,
-            IPolicyFragmentsExtractor policyFragmentsExtractor = null)
+            IPolicyFragmentsExtractor policyFragmentsExtractor = null,
+            IApiReleaseExtractor apiReleaseExtractor = null)
         => new ExtractorExecutor(
                 logger,
                 apisClient,
@@ -176,7 +181,8 @@ public static ExtractorExecutor BuildExtractorExecutor(
                 apiManagementServiceExtractor,
                 schemaExtractor,
                 openIdConnectProviderExtractor,
-                policyFragmentsExtractor);
+                policyFragmentsExtractor,
+                apiReleaseExtractor);
 
         public void SetExtractorParameters(ExtractorParameters extractorParameters)
         {
@@ -478,7 +484,8 @@ public async Task GenerateResourceParametersFiles(
             Template<IdentityProviderResources> identityProviderTemplate = null,
             Template<SchemaTemplateResources> schemaTemplate = null,
             Template<OpenIdConnectProviderResources> openIdConnectProviderTemplate = null,
-            Template<PolicyFragmentsResources> policyFragmentsTemplate = null)
+            Template<PolicyFragmentsResources> policyFragmentsTemplate = null,
+            Template<ApiReleaseTemplateResources> apiReleaseTemplate = null)
         {
             this.RenameExistingParametersDirectory(baseFilesGenerationDirectory);
 
@@ -498,6 +505,7 @@ public async Task GenerateResourceParametersFiles(
             await this.GenerateResourceParametersFile(baseFilesGenerationDirectory, this.extractorParameters.FileNames.SchemaParameters, schemaTemplate, mainParametersTemplate);
             await this.GenerateResourceParametersFile(baseFilesGenerationDirectory, this.extractorParameters.FileNames.OpenIdConnectProvidersParameters, openIdConnectProviderTemplate, mainParametersTemplate);
             await this.GenerateResourceParametersFile(baseFilesGenerationDirectory, this.extractorParameters.FileNames.PolicyFragmentsParameters, policyFragmentsTemplate, mainParametersTemplate);
+            await this.GenerateResourceParametersFile(baseFilesGenerationDirectory, this.extractorParameters.FileNames.ApiReleaseParameters, apiReleaseTemplate, mainParametersTemplate);
         }
 
         public async Task GenerateResourceParametersFile<TTemplateResource>(string baseFilesGenerationDirectory, string fileName, Template<TTemplateResource> resourceTemplate, Template mainParametersTemplate) where TTemplateResource : ITemplateResources, new()
@@ -901,6 +909,57 @@ await FileWriter.SaveAsJsonAsync(
         }
 
         /// <summary>
+        /// Generates api release template in the desired folder
+        /// </summary>
+        /// <param name="baseFilesGenerationDirectory">name of base folder where to save output files</param>
+        /// <returns>generated api release template</returns>
+        public async Task<Template<ApiReleaseTemplateResources>> GenerateApiReleaseTemplateAsync(string apiName, string baseFilesGenerationDirectory)
+        {
+            this.logger.LogInformation("Started generation of api release template...");
+
+            var apiReleaseTemplate = this.apiReleaseExtractor.GenerateSingleApiReleaseTemplate(apiName, this.extractorParameters);
+
+            if (apiReleaseTemplate?.HasResources() == true)
+            {
+                await FileWriter.SaveAsJsonAsync(
+                    apiReleaseTemplate,
+                    directory: baseFilesGenerationDirectory,
+                    fileName: this.extractorParameters.FileNames.ApiRelease);
+            }
+
+            this.logger.LogInformation("Finished generation of apiRelease template...");
+            return apiReleaseTemplate;
+        }
+
+        /// <summary>
+        /// Generates api release template in the desired folder
+        /// </summary>
+        /// <param name="baseFilesGenerationDirectory">name of base folder where to save output files</param>
+        /// <returns>generated api release template</returns>
+        public async Task<Template<ApiReleaseTemplateResources>> GenerateApiReleasesTemplateAsync(string baseFilesGenerationDirectory)
+        {
+            if (!string.IsNullOrEmpty(this.extractorParameters.SingleApiName) || !this.extractorParameters.MultipleApiNames.IsNullOrEmpty())
+            {
+                this.logger.LogInformation("Skip generation of api releases template: not all apis are extracted");
+                return null;
+            }
+
+            this.logger.LogInformation("Started generation of api releases template...");
+
+            var apiReleaseTemplate = await this.apiReleaseExtractor.GenerateCurrentApiReleaseTemplate(this.extractorParameters);
+
+            if (apiReleaseTemplate?.HasResources() == true)
+            {
+                await FileWriter.SaveAsJsonAsync(
+                    apiReleaseTemplate,
+                    directory: baseFilesGenerationDirectory,
+                    fileName: this.extractorParameters.FileNames.ApiRelease);
+            }
+
+            this.logger.LogInformation("Finished generation of apiReleases template...");
+            return apiReleaseTemplate;
+        }
+
         /// Generates policy fragments templates in the desired folder
         /// </summary>
         /// <param name="baseFilesGenerationDirectory">name of base folder where to save output files</param>
@@ -1034,14 +1093,17 @@ async Task GenerateSingleAPIWithRevisionsTemplates()
             this.logger.LogInformation("Extracting singleAPI {0} with revisions", this.extractorParameters.SingleApiName);
 
             string currentRevision = null;
+            bool generateSingleApiReleaseTemplate;
             List<string> revList = new List<string>();
 
             await foreach (var apiRevision in this.apiRevisionExtractor.GetApiRevisionsAsync(this.extractorParameters.SingleApiName, this.extractorParameters))
             {
+                generateSingleApiReleaseTemplate = false;
                 var apiRevisionName = apiRevision.ApiId.Split("/")[2];
                 if (apiRevision.IsCurrent)
                 {
                     currentRevision = apiRevisionName;
+                    generateSingleApiReleaseTemplate = true;
                 }
 
                 // creating a folder for this api revision
@@ -1050,6 +1112,11 @@ async Task GenerateSingleAPIWithRevisionsTemplates()
                 revList.Add(apiRevisionName);
 
                 await this.GenerateTemplates(revFileFolder, singleApiName: apiRevisionName);
+                
+                if (generateSingleApiReleaseTemplate)
+                {
+                    await this.GenerateApiReleaseTemplateAsync(apiRevisionName, revFileFolder);
+                }
             }
 
             if (currentRevision is null)
@@ -1088,9 +1155,9 @@ async Task GenerateTemplates(
             }
 
             var apisToExtract = await this.GetApiNamesToExtract(singleApiName, multipleApiNames);
-
             // generate different templates using extractors and write to output
             apiTemplate = apiTemplate ?? await this.GenerateApiTemplateAsync(singleApiName, multipleApiNames, baseFilesGenerationDirectory);
+            
             var globalServicePolicyTemplate = await this.GeneratePolicyTemplateAsync(baseFilesGenerationDirectory);
             var productApiTemplate = await this.GenerateProductApisTemplateAsync(singleApiName, multipleApiNames, baseFilesGenerationDirectory);
             var productTemplate = await this.GenerateProductsTemplateAsync(singleApiName, baseFilesGenerationDirectory, apiTemplate.TypedResources.ApiProducts);
@@ -1106,9 +1173,11 @@ async Task GenerateTemplates(
             var openIdConnectProviderTemplate = await this.GenerateOpenIdConnectProviderTemplateAsync(baseFilesGenerationDirectory);
             var schemasTempate = await this.GenerateSchemasTemplateAsync(baseFilesGenerationDirectory);
             var policyFragmentTemplate = await this.GeneratePolicyFragmentsTemplateAsync(apiTemplate.TypedResources.GetAllPolicies(), baseFilesGenerationDirectory);
+            var apiReleasesTemplate = await this.GenerateApiReleasesTemplateAsync(baseFilesGenerationDirectory);
             await this.GenerateGatewayTemplateAsync(singleApiName, baseFilesGenerationDirectory);
             await this.GenerateGatewayApiTemplateAsync(singleApiName, multipleApiNames, baseFilesGenerationDirectory);
             await this.GenerateApiManagementServiceTemplate(baseFilesGenerationDirectory);
+            
             var parametersTemplate = await this.GenerateParametersTemplateAsync(apisToExtract, loggerTemplate.TypedResources, backendTemplate.TypedResources, namedValueTemplate.TypedResources, identityProviderTemplate.TypedResources, openIdConnectProviderTemplate.TypedResources, baseFilesGenerationDirectory);
 
             await this.GenerateResourceParametersFiles(
@@ -1129,7 +1198,8 @@ await this.GenerateResourceParametersFiles(
                 identityProviderTemplate: identityProviderTemplate,
                 openIdConnectProviderTemplate: openIdConnectProviderTemplate,
                 schemaTemplate: schemasTempate,
-                policyFragmentsTemplate: policyFragmentTemplate);
+                policyFragmentsTemplate: policyFragmentTemplate,
+                apiReleaseTemplate: apiReleasesTemplate);
 
             await this.GenerateMasterTemplateAsync(
                 baseFilesGenerationDirectory,

src/ArmTemplates/Common/API/Clients/Abstractions/IApisClient.cs

@@ -16,6 +16,8 @@ public interface IApisClient
 
         Task<List<ApiTemplateResource>> GetAllAsync(ExtractorParameters extractorParameters);
 
+        Task<List<ApiTemplateResource>> GetAllCurrentAsync(ExtractorParameters extractorParameters);
+
         Task<List<ApiTemplateResource>> GetAllLinkedToProductAsync(string productName, ExtractorParameters extractorParameters);
 
         Task<List<ApiTemplateResource>> GetAllLinkedToGatewayAsync(string gatewayName, ExtractorParameters extractorParameters);

src/ArmTemplates/Common/API/Clients/Apis/ApisClient.cs

@@ -10,18 +10,23 @@
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Constants;
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.Apis;
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Extractor.Models;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Extractor.Utilities.DataProcessors.Absctraction;
 
 namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.API.Clients.Apis
 {
     public class ApisClient : ApiClientBase, IApisClient
     {
         const string GetSingleApiRequest = "{0}/subscriptions/{1}/resourceGroups/{2}/providers/Microsoft.ApiManagement/service/{3}/apis/{4}?api-version={5}";
         const string GetAllApisRequest = "{0}/subscriptions/{1}/resourceGroups/{2}/providers/Microsoft.ApiManagement/service/{3}/apis?api-version={4}";
+        const string GetAllCurrentApisRequest = "{0}/subscriptions/{1}/resourceGroups/{2}/providers/Microsoft.ApiManagement/service/{3}/apis?api-version={4}&$filter=isCurrent";
         const string GetAllApisLinkedToProductRequest = "{0}/subscriptions/{1}/resourceGroups/{2}/providers/Microsoft.ApiManagement/service/{3}/products/{4}/apis?api-version={5}";
         const string GetApisLinkedToGatewayRequest = "{0}/subscriptions/{1}/resourceGroups/{2}/providers/Microsoft.ApiManagement/service/{3}/gateways/{4}/apis?api-version={5}";
 
-        public ApisClient(IHttpClientFactory httpClientFactory) : base(httpClientFactory)
+        readonly IApiDataProcessor apiDataProcessor;
+
+        public ApisClient(IHttpClientFactory httpClientFactory, IApiDataProcessor apiDataProcessor) : base(httpClientFactory)
         {
+            this.apiDataProcessor = apiDataProcessor;
         }
 
         public async Task<ApiTemplateResource> GetSingleAsync(string apiName, ExtractorParameters extractorParameters)
@@ -30,7 +35,10 @@ public async Task<ApiTemplateResource> GetSingleAsync(string apiName, ExtractorP
             string requestUrl = string.Format(GetSingleApiRequest,
                 this.BaseUrl, azSubId, extractorParameters.ResourceGroup, extractorParameters.SourceApimName, apiName, GlobalConstants.ApiVersion);
 
-            return await this.GetResponseAsync<ApiTemplateResource>(azToken, requestUrl);
+            var api = await this.GetResponseAsync<ApiTemplateResource>(azToken, requestUrl);
+            this.apiDataProcessor.ProcessSingleData(api);
+
+            return api;
         }
 
         public async Task<List<ApiTemplateResource>> GetAllAsync(ExtractorParameters extractorParameters)
@@ -40,7 +48,23 @@ public async Task<List<ApiTemplateResource>> GetAllAsync(ExtractorParameters ext
             string requestUrl = string.Format(GetAllApisRequest,
                 this.BaseUrl, azSubId, extractorParameters.ResourceGroup, extractorParameters.SourceApimName, GlobalConstants.ApiVersion);
 
-            return await this.GetPagedResponseAsync<ApiTemplateResource>(azToken, requestUrl);
+            var apis = await this.GetPagedResponseAsync<ApiTemplateResource>(azToken, requestUrl);
+            this.apiDataProcessor.ProcessData(apis);
+            
+            return apis;
+        }
+
+        public async Task<List<ApiTemplateResource>> GetAllCurrentAsync(ExtractorParameters extractorParameters)
+        {
+            var (azToken, azSubId) = await this.Auth.GetAccessToken();
+
+            string requestUrl = string.Format(GetAllCurrentApisRequest,
+                this.BaseUrl, azSubId, extractorParameters.ResourceGroup, extractorParameters.SourceApimName, GlobalConstants.ApiVersion);
+
+            var apis = await this.GetPagedResponseAsync<ApiTemplateResource>(azToken, requestUrl);
+            this.apiDataProcessor.ProcessData(apis);
+
+            return apis;
         }
 
         public async Task<List<ApiTemplateResource>> GetAllLinkedToProductAsync(string productName, ExtractorParameters extractorParameters)
@@ -49,7 +73,10 @@ public async Task<List<ApiTemplateResource>> GetAllLinkedToProductAsync(string p
             string requestUrl = string.Format(GetAllApisLinkedToProductRequest,
                 this.BaseUrl, azSubId, extractorParameters.ResourceGroup, extractorParameters.SourceApimName, productName, GlobalConstants.ApiVersion);
 
-            return await this.GetPagedResponseAsync<ApiTemplateResource>(azToken, requestUrl);
+            var apis = await this.GetPagedResponseAsync<ApiTemplateResource>(azToken, requestUrl);
+            this.apiDataProcessor.ProcessData(apis);
+
+            return apis;
         }
 
         public async Task<List<ApiTemplateResource>> GetAllLinkedToGatewayAsync(string gatewayName, ExtractorParameters extractorParameters)
@@ -58,7 +85,10 @@ public async Task<List<ApiTemplateResource>> GetAllLinkedToGatewayAsync(string g
             string requestUrl = string.Format(GetApisLinkedToGatewayRequest,
                 this.BaseUrl, azSubId, extractorParameters.ResourceGroup, extractorParameters.SourceApimName, gatewayName, GlobalConstants.ApiVersion);
 
-            return await this.GetPagedResponseAsync<ApiTemplateResource>(azToken, requestUrl);
+            var apis = await this.GetPagedResponseAsync<ApiTemplateResource>(azToken, requestUrl);
+            this.apiDataProcessor.ProcessData(apis);
+
+            return apis;
         }
     }
 }

src/ArmTemplates/Common/Extensions/NamingHelper.cs

@@ -5,6 +5,7 @@
 
 using System.Linq;
 using System.Text.RegularExpressions;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Constants;
 
 namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Extensions
 {
@@ -13,6 +14,11 @@ public static class NamingHelper
         static readonly Regex ExcludeOtherFromLettersAndDigitsRegex = new Regex("[^a-zA-Z0-9]");
         static readonly Regex ExcludeOtherFromAlphaNumericsAndHyphensRegex = new Regex("[^a-zA-Z0-9-]");
 
+        // ValidDeployment name is the string following the pattern: '^[-\w\._\(\)]+$'. https://docs.microsoft.com/en-us/rest/api/resources/deployments/create-or-update?tabs=HTTP#uri-parameters
+        // ExcludeOtherFromValidDeploymentNameCharsRegex matches any character other than [alphanumerics + '-' + '.' + '(' + ')' + '_'];
+        // Example: 'template;rev=1.json' string will match the ';' and '=' chars
+        static readonly Regex ExcludeOtherFromValidDeploymentNameCharsRegex = new Regex(@"[^-\w\._\(\)]"); 
+
         public static string GetSubstringBetweenTwoCharacters(char left, char right, string fullString)
         {
             var regex = new Regex($"(?<={left})(.*?)(?={right})");
@@ -60,5 +66,21 @@ public static string GenerateParametrizedResourceName(string parameterName, stri
         {
             return $"[concat(parameters('{parameterName}'), '/{resourceName}')]";
         }
+
+        public static string GenerateValidDeploymentFileName(string fileName)
+        {
+            if (string.IsNullOrEmpty(fileName))
+            {
+                return string.Empty;
+            }
+
+            var resourceName = ExcludeOtherFromValidDeploymentNameCharsRegex.Replace(fileName, "-");
+            return resourceName;
+        }
+
+        public static string GenerateApisResourceId(string apiName)
+        {
+            return $"[resourceId('{ResourceTypeConstants.API}', parameters('{ParameterNames.ApimServiceName}'), '{apiName}')]";
+        }
     }
 }