Merge pull request #60 from delegateas/features/webresource-analyzer

WebResource analyzer

Author: Lucki2g

Generator/DTO/Warnings/AttributeWarning.cs

@@ -0,0 +1,6 @@
+namespace Generator.DTO.Warnings;
+
+public record AttributeWarning(string Message) : SolutionWarning(
+        SolutionWarningType.Attribute,
+        Message
+    );

Generator/DTO/Warnings/SolutionWarning.cs

@@ -0,0 +1,11 @@
+namespace Generator.DTO.Warnings;
+
+public enum SolutionWarningType
+{
+    Attribute,
+}
+
+public record SolutionWarning(
+        SolutionWarningType Type,
+        string Message
+    );

Generator/DTO/WebResource.cs

@@ -0,0 +1,10 @@
+using Microsoft.Xrm.Sdk;
+
+namespace Generator.DTO;
+
+public record WebResource(
+    string Id,
+    string Name,
+    string Content,
+    OptionSetValue WebResourceType,
+    string? Description = null) : Analyzeable();

Generator/DataverseService.cs

@@ -2,9 +2,11 @@
 using Azure.Identity;
 using Generator.DTO;
 using Generator.DTO.Attributes;
+using Generator.DTO.Warnings;
 using Generator.Queries;
 using Generator.Services;
 using Generator.Services.Plugins;
+using Generator.Services.WebResources;
 using Microsoft.Crm.Sdk.Messages;
 using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.Configuration;
@@ -14,6 +16,7 @@
 using Microsoft.Xrm.Sdk.Metadata;
 using Microsoft.Xrm.Sdk.Query;
 using System.Collections.Concurrent;
+using System.Diagnostics;
 using System.Reflection;
 using Attribute = Generator.DTO.Attributes.Attribute;
 
@@ -27,6 +30,7 @@ internal class DataverseService
 
         private readonly PluginAnalyzer pluginAnalyzer;
         private readonly PowerAutomateFlowAnalyzer flowAnalyzer;
+        private readonly WebResourceAnalyzer webResourceAnalyzer;
 
         public DataverseService(IConfiguration configuration, ILogger<DataverseService> logger)
         {
@@ -47,10 +51,12 @@ public DataverseService(IConfiguration configuration, ILogger<DataverseService>
 
             pluginAnalyzer = new PluginAnalyzer(client);
             flowAnalyzer = new PowerAutomateFlowAnalyzer(client);
+            webResourceAnalyzer = new WebResourceAnalyzer(client, configuration);
         }
 
-        public async Task<IEnumerable<Record>> GetFilteredMetadata()
+        public async Task<(IEnumerable<Record>, IEnumerable<SolutionWarning>)> GetFilteredMetadata()
         {
+            var warnings = new List<SolutionWarning>(); // used to collect warnings for the insights dashboard
             var (publisherPrefix, solutionIds) = await GetSolutionIds();
             var solutionComponents = await GetSolutionComponents(solutionIds); // (id, type, rootcomponentbehavior)
 
@@ -96,15 +102,32 @@ public async Task<IEnumerable<Record>> GetFilteredMetadata()
             // Processes analysis
             var attributeUsages = new Dictionary<string, Dictionary<string, List<AttributeUsage>>>();
             // Plugins
+            var pluginStopWatch = new Stopwatch();
+            pluginStopWatch.Start();
             var pluginCollection = await client.GetSDKMessageProcessingStepsAsync(solutionIds);
             logger.LogInformation($"There are {pluginCollection.Count()} plugin sdk steps in the environment.");
             foreach (var plugin in pluginCollection)
                 await pluginAnalyzer.AnalyzeComponentAsync(plugin, attributeUsages);
+            pluginStopWatch.Stop();
+            logger.LogInformation($"Plugin analysis took {pluginStopWatch.ElapsedMilliseconds} ms.");
             // Flows
+            var flowStopWatch = new Stopwatch();
+            flowStopWatch.Start();
             var flowCollection = await client.GetPowerAutomateFlowsAsync(solutionIds);
             logger.LogInformation($"There are {flowCollection.Count()} Power Automate flows in the environment.");
             foreach (var flow in flowCollection)
                 await flowAnalyzer.AnalyzeComponentAsync(flow, attributeUsages);
+            flowStopWatch.Stop();
+            logger.LogInformation($"Power Automate flow analysis took {flowStopWatch.ElapsedMilliseconds} ms.");
+            // WebResources
+            var resourceStopWatch = new Stopwatch();
+            resourceStopWatch.Start();
+            var webresourceCollection = await client.GetWebResourcesAsync(solutionIds);
+            logger.LogInformation($"There are {webresourceCollection.Count()} WebResources in the environment.");
+            foreach (var resource in webresourceCollection)
+                await webResourceAnalyzer.AnalyzeComponentAsync(resource, attributeUsages);
+            resourceStopWatch.Stop();
+            logger.LogInformation($"WebResource analysis took {resourceStopWatch.ElapsedMilliseconds} ms.");
 
             var records =
                 entitiesInSolutionMetadata
@@ -123,8 +146,17 @@ public async Task<IEnumerable<Record>> GetFilteredMetadata()
                 .Where(x => x.EntityMetadata.DisplayName.UserLocalizedLabel?.Label != null)
                 .ToList();
 
+            // Warn about attributes that were used in processes, but the entity could not be resolved from e.g. JavaScript file name or similar
+            var hash = entitiesInSolutionMetadata.SelectMany<EntityMetadata, string>(r => [r.LogicalCollectionName?.ToLower() ?? "", r.LogicalName.ToLower()]).ToHashSet();
+            warnings.AddRange(attributeUsages.Keys
+                .Where(k => !hash.Contains(k.ToLower()))
+                .SelectMany(entityKey => attributeUsages.GetValueOrDefault(entityKey)!
+                    .SelectMany(attributeDict => attributeDict.Value
+                        .Select(usage =>
+                            new AttributeWarning($"{attributeDict.Key} was used inside a {usage.ComponentType} component [{usage.Name}]. However, the entity {entityKey} could not be resolved in the provided solutions.")))));
 
-            return records
+
+            return (records
                 .Select(x =>
                 {
                     logicalNameToSecurityRoles.TryGetValue(x.EntityMetadata.LogicalName, out var securityRoles);
@@ -142,7 +174,7 @@ public async Task<IEnumerable<Record>> GetFilteredMetadata()
                         entityIconMap,
                         attributeUsages,
                         configuration);
-                });
+                }), warnings);
         }
 
         private static Record MakeRecord(

Generator/Generator.csproj

@@ -13,6 +13,7 @@
     <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="9.0.0" />
     <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.6" />
     <PackageReference Include="Microsoft.PowerPlatform.Dataverse.Client" Version="1.2.2" />
+    <PackageReference Include="System.Linq.Dynamic.Core" Version="1.6.7" />
   </ItemGroup>
 
   <ItemGroup>

Generator/Program.cs

@@ -18,8 +18,8 @@
 var logger = loggerFactory.CreateLogger<DataverseService>();
 
 var dataverseService = new DataverseService(configuration, logger);
-var entities = (await dataverseService.GetFilteredMetadata()).ToList();
+var (entities, warnings) = await dataverseService.GetFilteredMetadata();
 
-var websiteBuilder = new WebsiteBuilder(configuration, entities);
+var websiteBuilder = new WebsiteBuilder(configuration, entities, warnings);
 websiteBuilder.AddData();
 

Generator/Queries/WebResourceQueries.cs

@@ -0,0 +1,84 @@
+using Generator.DTO;
+using Microsoft.PowerPlatform.Dataverse.Client;
+using Microsoft.Xrm.Sdk;
+using Microsoft.Xrm.Sdk.Query;
+
+namespace Generator.Queries;
+
+public static class WebResourceQueries
+{
+
+    public static async Task<IEnumerable<WebResource>> GetWebResourcesAsync(this ServiceClient service, List<Guid>? solutionIds = null)
+    {
+        var query = new QueryExpression("solutioncomponent")
+        {
+            ColumnSet = new ColumnSet("objectid"),
+            Criteria = new FilterExpression(LogicalOperator.And)
+            {
+                Conditions =
+                {
+                    new ConditionExpression("solutionid", ConditionOperator.In, solutionIds),
+                    new ConditionExpression("componenttype", ConditionOperator.Equal, 61) // 61 = Web Resource
+                }
+            },
+            LinkEntities =
+            {
+                new LinkEntity(
+                    "solutioncomponent",
+                    "webresource",
+                    "objectid",
+                    "webresourceid",
+                    JoinOperator.Inner)
+                {
+                    Columns = new ColumnSet("webresourceid", "name", "content", "webresourcetype", "description"),
+                    EntityAlias = "webresource",
+                    LinkCriteria = new FilterExpression
+                    {
+                        Conditions =
+                        {
+                            new ConditionExpression("webresourcetype", ConditionOperator.Equal, 3) // JS Resources
+                        }
+                    }
+                }
+            }
+        };
+
+        var results = (await service.RetrieveMultipleAsync(query)).Entities;
+
+        var webResources = results.Select(e =>
+        {
+            var content = "";
+            var contentValue = e.GetAttributeValue<AliasedValue>("webresource.content")?.Value;
+            var webresourceId = e.GetAttributeValue<AliasedValue>("webresource.webresourceid").Value?.ToString() ?? "";
+            var webresourceName = e.GetAttributeValue<AliasedValue>("webresource.name").Value?.ToString();
+            if (contentValue != null)
+            {
+                // Content is base64 encoded, decode it
+                var base64Content = contentValue.ToString();
+                if (!string.IsNullOrEmpty(base64Content))
+                {
+                    try
+                    {
+                        var bytes = Convert.FromBase64String(base64Content);
+                        content = System.Text.Encoding.UTF8.GetString(bytes);
+                    }
+                    catch
+                    {
+                        // If decoding fails, keep the base64 content
+                        content = base64Content;
+                    }
+                }
+            }
+
+            return new WebResource(
+                webresourceId,
+                webresourceName,
+                content,
+                (OptionSetValue)e.GetAttributeValue<AliasedValue>("webresource.webresourcetype").Value,
+                e.GetAttributeValue<AliasedValue>("webresource.description")?.Value?.ToString()
+            );
+        });
+
+        return webResources;
+    }
+}

Generator/Services/ComponentAnalyzerBase.cs

@@ -11,6 +11,22 @@ public abstract class BaseComponentAnalyzer<T>(ServiceClient service) : ICompone
     public abstract ComponentType SupportedType { get; }
     public abstract Task AnalyzeComponentAsync(T component, Dictionary<string, Dictionary<string, List<AttributeUsage>>> attributeUsages);
 
+    protected void AddAttributeUsage(Dictionary<string, Dictionary<string, List<AttributeUsage>>> attributeUsages,
+        string entityName, string attributeName, AttributeUsage usage)
+    {
+        if (!attributeUsages.ContainsKey(entityName))
+        {
+            attributeUsages[entityName] = new Dictionary<string, List<AttributeUsage>>();
+        }
+
+        if (!attributeUsages[entityName].ContainsKey(attributeName))
+        {
+            attributeUsages[entityName][attributeName] = new List<AttributeUsage>();
+        }
+
+        attributeUsages[entityName][attributeName].Add(usage);
+    }
+
     protected List<string> ExtractFieldsFromODataFilter(string filter)
     {
         var fields = new List<string>();

Generator/Services/Plugins/PluginAnalyzer.cs

@@ -23,17 +23,8 @@ public override async Task AnalyzeComponentAsync(SDKStep sdkStep, Dictionary<str
 
             // Populate the attributeUsages dictionary
             foreach (var attribute in filteringAttributes)
-            {
-                if (!attributeUsages.ContainsKey(logicalTableName))
-                    attributeUsages[logicalTableName] = new Dictionary<string, List<AttributeUsage>>();
+                AddAttributeUsage(attributeUsages, logicalTableName, attribute, new AttributeUsage(pluginName, $"Used in filterattributes", OperationType.Other, SupportedType));
 
-                if (!attributeUsages[logicalTableName].ContainsKey(attribute))
-                    attributeUsages[logicalTableName][attribute] = new List<AttributeUsage>();
-
-                // Add the usage information (assuming AttributeUsage is a defined class)
-
-                attributeUsages[logicalTableName][attribute].Add(new AttributeUsage(pluginName, $"Used in filterattributes", OperationType.Other, SupportedType));
-            }
         }
         catch (Exception ex)
         {

Generator/Services/Power Automate/PowerAutomateFlowAnalyzer.cs

@@ -448,20 +448,4 @@ private OperationType DetermineOperationTypeFromAction(string actionName)
         // Default to Other for unknown actions
         return OperationType.Other;
     }
-
-    private void AddAttributeUsage(Dictionary<string, Dictionary<string, List<AttributeUsage>>> attributeUsages,
-        string entityName, string attributeName, AttributeUsage usage)
-    {
-        if (!attributeUsages.ContainsKey(entityName))
-        {
-            attributeUsages[entityName] = new Dictionary<string, List<AttributeUsage>>();
-        }
-
-        if (!attributeUsages[entityName].ContainsKey(attributeName))
-        {
-            attributeUsages[entityName][attributeName] = new List<AttributeUsage>();
-        }
-
-        attributeUsages[entityName][attributeName].Add(usage);
-    }
 }