Feature/extract/policy fragments (#775)

* Add policy-fragments extraction

* Add policy fragment tests, update main template tests

Co-authored-by: Farhad Alizada <falizada@microsoft.com>

Author: f-alizada

src/ArmTemplates/Commands/Executors/ExtractorExecutor.cs

@@ -24,6 +24,7 @@
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.NamedValues;
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.OpenIdConnectProviders;
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.Policy;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.PolicyFragments;
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.ProductApis;
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.Products;
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.Schemas;
@@ -68,6 +69,7 @@ public class ExtractorExecutor
         readonly IApiManagementServiceExtractor apiManagementServiceExtractor;
         readonly ISchemaExtractor schemaExtractor;
         readonly IOpenIdConnectProviderExtractor openIdConnectProviderExtractor;
+        readonly IPolicyFragmentsExtractor policyFragmentsExtractor;
 
         public ExtractorExecutor(
             ILogger<ExtractorExecutor> logger,
@@ -92,7 +94,8 @@ public ExtractorExecutor(
             IIdentityProviderExtractor identityProviderExtractor,
             IApiManagementServiceExtractor apiManagementServiceExtractor,
             ISchemaExtractor schemaExtractor,
-            IOpenIdConnectProviderExtractor openIdConnectProviderExtractor)
+            IOpenIdConnectProviderExtractor openIdConnectProviderExtractor,
+            IPolicyFragmentsExtractor policyFragmentsExtractor)
         {
             this.logger = logger;
             this.apisClient = apisClient;
@@ -117,6 +120,7 @@ public ExtractorExecutor(
             this.apiManagementServiceExtractor = apiManagementServiceExtractor;
             this.schemaExtractor = schemaExtractor;
             this.openIdConnectProviderExtractor = openIdConnectProviderExtractor;
+            this.policyFragmentsExtractor = policyFragmentsExtractor;
         }
 
         /// <summary>
@@ -146,7 +150,8 @@ public static ExtractorExecutor BuildExtractorExecutor(
             IIdentityProviderExtractor identityProviderExtractor = null,
             IApiManagementServiceExtractor apiManagementServiceExtractor = null,
             ISchemaExtractor schemaExtractor = null,
-            IOpenIdConnectProviderExtractor openIdConnectProviderExtractor = null)
+            IOpenIdConnectProviderExtractor openIdConnectProviderExtractor = null,
+            IPolicyFragmentsExtractor policyFragmentsExtractor = null)
         => new ExtractorExecutor(
                 logger,
                 apisClient,
@@ -170,7 +175,8 @@ public static ExtractorExecutor BuildExtractorExecutor(
                 identityProviderExtractor,
                 apiManagementServiceExtractor,
                 schemaExtractor,
-                openIdConnectProviderExtractor);
+                openIdConnectProviderExtractor,
+                policyFragmentsExtractor);
 
         public void SetExtractorParameters(ExtractorParameters extractorParameters)
         {
@@ -469,7 +475,8 @@ public async Task<Template<MasterTemplateResources>> GenerateMasterTemplateAsync
            GroupTemplateResources groupTemplateResources = null,
            IdentityProviderResources identityProviderTemplateResources = null,
            SchemaTemplateResources schemaTemplateResources = null,
-           OpenIdConnectProviderResources openIdConnectProviderResources = null)
+           OpenIdConnectProviderResources openIdConnectProviderResources = null,
+           PolicyFragmentsResources policyFragmentsResources = null)
         {
             if (string.IsNullOrEmpty(this.extractorParameters.LinkedTemplatesBaseUrl))
             {
@@ -483,7 +490,7 @@ public async Task<Template<MasterTemplateResources>> GenerateMasterTemplateAsync
                 this.extractorParameters, apiTemplateResources, policyTemplateResources, apiVersionSetTemplateResources,
                 productsTemplateResources, productApisTemplateResources, apiTagsTemplateResources, loggersTemplateResources,
                 backendsTemplateResources, authorizationServersTemplateResources, namedValuesTemplateResources, tagTemplateResources, 
-                groupTemplateResources, identityProviderTemplateResources, schemaTemplateResources, openIdConnectProviderResources);
+                groupTemplateResources, identityProviderTemplateResources, schemaTemplateResources, openIdConnectProviderResources, policyFragmentsResources);
 
             if (masterTemplate?.HasResources() == true)
             {
@@ -813,6 +820,29 @@ await FileWriter.SaveAsJsonAsync(
             return schemasTemplate;
         }
 
+        /// <summary>
+        /// Generates policy fragments templates in the desired folder
+        /// </summary>
+        /// <param name="baseFilesGenerationDirectory">name of base folder where to save output files</param>
+        /// <returns>generated policy fragments template</returns>
+        public async Task<Template<PolicyFragmentsResources>> GeneratePolicyFragmentsTemplateAsync(List<PolicyTemplateResource> apiPolicies, string baseFilesGenerationDirectory)
+        {
+            this.logger.LogInformation("Started generation of policy fragments template...");
+
+            var policyFragmentTemplate = await this.policyFragmentsExtractor.GeneratePolicyFragmentsTemplateAsync(apiPolicies, this.extractorParameters);
+
+            if (policyFragmentTemplate?.HasResources() == true)
+            {
+                await FileWriter.SaveAsJsonAsync(
+                    policyFragmentTemplate,
+                    directory: baseFilesGenerationDirectory,
+                    fileName: this.extractorParameters.FileNames.PolicyFragments);
+            }
+
+            this.logger.LogInformation("Finished generation of policy fragments template...");
+            return policyFragmentTemplate;
+        }
+
         /// <summary>
         /// Generates split api templates / folders for each api in this sourceApim 
         /// </summary>
@@ -995,6 +1025,7 @@ async Task GenerateTemplates(
             var identityProviderTemplate = await this.GenerateIdentityProviderTemplateAsync(baseFilesGenerationDirectory);
             var openIdConnectProviderTemplate = await this.GenerateOpenIdConnectProviderTemplateAsync(baseFilesGenerationDirectory);
             var schemasTempate = await this.GenerateSchemasTemplateAsync(baseFilesGenerationDirectory);
+            var policyFragmentTemplate = await this.GeneratePolicyFragmentsTemplateAsync(apiTemplate.TypedResources.GetAllPolicies(), baseFilesGenerationDirectory);
             await this.GenerateGatewayTemplateAsync(singleApiName, baseFilesGenerationDirectory);
             await this.GenerateGatewayApiTemplateAsync(singleApiName, multipleApiNames, baseFilesGenerationDirectory);
             await this.GenerateApiManagementServiceTemplate(baseFilesGenerationDirectory);
@@ -1016,7 +1047,8 @@ await this.GenerateMasterTemplateAsync(
                 groupTemplateResources: groupTemplate.TypedResources,
                 identityProviderTemplateResources: identityProviderTemplate.TypedResources,
                 schemaTemplateResources: schemasTempate.TypedResources,
-                openIdConnectProviderResources: openIdConnectProviderTemplate.TypedResources);
+                openIdConnectProviderResources: openIdConnectProviderTemplate.TypedResources,
+                policyFragmentsResources: policyFragmentTemplate.TypedResources);
         }
 
 

src/ArmTemplates/Common/API/Clients/Abstractions/IPolicyFragmentsClient.cs

@@ -0,0 +1,17 @@
+// --------------------------------------------------------------------------
+//  Copyright (c) Microsoft Corporation. All rights reserved.
+//  Licensed under the MIT License.
+// --------------------------------------------------------------------------
+
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.PolicyFragments;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Extractor.Models;
+
+namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.API.Clients.Abstractions
+{
+    public interface IPolicyFragmentsClient
+    {
+        Task<List<PolicyFragmentsResource>> GetAllAsync(ExtractorParameters extractorParameters);
+    }
+}

src/ArmTemplates/Common/API/Clients/PolicyFragments/PolicyFragmentsClient.cs

@@ -0,0 +1,43 @@
+// --------------------------------------------------------------------------
+//  Copyright (c) Microsoft Corporation. All rights reserved.
+//  Licensed under the MIT License.
+// --------------------------------------------------------------------------
+
+using System.Collections.Generic;
+using System.Net.Http;
+using System.Threading.Tasks;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.API.Clients.Abstractions;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Constants;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.PolicyFragments;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Extractor.Models;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Extractor.Utilities.DataProcessors.Absctraction;
+
+namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.API.Clients.PolicyFragments
+{
+    public class PolicyFragmentsClient : ApiClientBase, IPolicyFragmentsClient
+    {
+        const string GetAllRequest = "{0}/subscriptions/{1}/resourceGroups/{2}/providers/Microsoft.ApiManagement/service/{3}/policyFragments?api-version={4}&format=rawxml";
+
+        readonly IPolicyFragmentDataProcessor policyFragmentDataProcessor;
+
+        public PolicyFragmentsClient(
+            IHttpClientFactory httpClientFactory,
+            IPolicyFragmentDataProcessor policyFragmentDataProcessor): base(httpClientFactory)
+        {
+            this.policyFragmentDataProcessor = policyFragmentDataProcessor;
+        }
+
+        public async Task<List<PolicyFragmentsResource>> GetAllAsync(ExtractorParameters extractorParameters)
+        {
+            var (azToken, azSubId) = await this.Auth.GetAccessToken();
+
+            var requestUrl = string.Format(GetAllRequest,
+                this.BaseUrl, azSubId, extractorParameters.ResourceGroup, extractorParameters.SourceApimName, GlobalConstants.ApiVersionPreview);
+
+            var policyFragments = await this.GetPagedResponseAsync<PolicyFragmentsResource>(azToken, requestUrl);
+            this.policyFragmentDataProcessor.ProcessData(policyFragments, extractorParameters);
+
+            return policyFragments;
+        }
+    }
+}

src/ArmTemplates/Common/Constants/GlobalConstants.cs

@@ -16,6 +16,7 @@ public static class GlobalConstants
         public const string ExtractDescription = "Extract an existing API Management instance";
 
         public const string ApiVersion = "2021-08-01";
+        public const string ApiVersionPreview = "2021-12-01-preview";
         public const string ArmApiVersion = "2022-01-01";
         public const int NumOfRecords = 100;
 

src/ArmTemplates/Common/Constants/ResourceTypeConstants.cs

@@ -38,5 +38,6 @@ public static class ResourceTypeConstants
         public const string ApiManagementService = "Microsoft.ApiManagement/service";
         public const string Schema = "Microsoft.ApiManagement/service/schemas";
         public const string OpenIdConnectProvider = "Microsoft.ApiManagement/service/openidConnectProviders";
+        public const string PolicyFragments = "Microsoft.ApiManagement/service/policyFragments";
     }
 }

src/ArmTemplates/Common/FileHandlers/FileNameGenerator.cs

@@ -39,6 +39,7 @@ public static FileNames GenerateFileNames(string baseFileName)
                 ApiManagementService = $@"{baseFileName}api-management-service.template.json",
                 TagApi = $@"{baseFileName}apiTags.template.json",
                 Schema = $@"{baseFileName}schemas.template.json",
+                PolicyFragments = $@"{baseFileName}policy-fragments.template.json",
                 Parameters = $@"{baseFileName}parameters.json",
                 LinkedMaster = $@"{baseFileName}master.template.json",
                 Apis = "/Apis",

src/ArmTemplates/Common/FileHandlers/FileNames.cs

@@ -41,6 +41,8 @@ public class FileNames
 
         public string Schema { get; set; }
 
+        public string PolicyFragments { get; set; }
+
         public string Parameters { get; set; }
 
         // linked property outputs 1 master template

src/ArmTemplates/Common/Templates/PolicyFragments/PolicyFragmentsProperties.cs

@@ -0,0 +1,16 @@
+// --------------------------------------------------------------------------
+//  Copyright (c) Microsoft Corporation. All rights reserved.
+//  Licensed under the MIT License.
+// --------------------------------------------------------------------------
+
+namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.PolicyFragments
+{
+    public class PolicyFragmentsProperties
+    {
+        public string Description { get; set; }
+
+        public string Format { get; set; }
+
+        public string Value { get; set; }
+    }
+}

src/ArmTemplates/Common/Templates/PolicyFragments/PolicyFragmentsResource.cs

@@ -0,0 +1,18 @@
+// --------------------------------------------------------------------------
+//  Copyright (c) Microsoft Corporation. All rights reserved.
+//  Licensed under the MIT License.
+// --------------------------------------------------------------------------
+
+using Newtonsoft.Json;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.Abstractions;
+
+namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.PolicyFragments
+{
+    public class PolicyFragmentsResource : TemplateResource
+    {
+        [JsonIgnore]
+        public string OriginalName { get; set; }
+
+        public PolicyFragmentsProperties Properties { get; set; }
+    }
+}

src/ArmTemplates/Common/Templates/PolicyFragments/PolicyFragmentsResources.cs

@@ -0,0 +1,26 @@
+// --------------------------------------------------------------------------
+//  Copyright (c) Microsoft Corporation. All rights reserved.
+//  Licensed under the MIT License.
+// --------------------------------------------------------------------------
+
+using System.Collections.Generic;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Extensions;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.Abstractions;
+
+namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.PolicyFragments
+{
+    public class PolicyFragmentsResources : ITemplateResources
+    {
+        public List<PolicyFragmentsResource> PolicyFragments { get; set; } = new();
+
+        public TemplateResource[] BuildTemplateResources()
+        {
+            return this.PolicyFragments.ToArray();
+        }
+
+        public bool HasContent()
+        {
+            return !this.PolicyFragments.IsNullOrEmpty();
+        }
+    }
+}