Memory and performance

Added our own reference detection and resolution.  Started removing projects form the in memory solution once we process them.  Added links from the solution to the projects.  Added links between partial classes.

Author: mkestler-rtp

DotNETDepends/Dependencies.cs

@@ -25,13 +25,9 @@ public DependencyEntry CreateProjectEntry(string filePath)
             return dependency;
         }
 
-        public DependencyEntry CreateCodeFileEntry(string filePath, SemanticModel semantic, SyntaxTree tree)
+        public DependencyEntry CreateCodeFileEntry(string filePath)
         {
-            var dependency = new DependencyEntry(filePath, EntryType.File)
-            {
-                Semantic = semantic,
-                Tree = tree
-            };
+            var dependency = new DependencyEntry(filePath, EntryType.File);
             entries[filePath] = dependency;
             return dependency;
         }

DotNETDepends/DependencyEntry.cs

@@ -1,9 +1,4 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis;
 
 namespace DotNETDepends
 {
@@ -21,28 +16,41 @@ class DependencyEntry
         public readonly String FilePath;
         public readonly EntryType Type;
         public HashSet<ISymbol> Symbols = new(SymbolEqualityComparer.Default);
-       //This is the Roslyn semantic model.  Only valid if Type == File.  
-        public SemanticModel? Semantic { get; set; }
-        //SyntaxTree of a C# or VB file.  Comes from Roslyn.
-        public SyntaxTree? Tree { get; set; }
+        private readonly HashSet<string> References = new();
+
         //File dependencies of the file
         public HashSet<string> Dependencies { get; } = new HashSet<string>();
 
         public DependencyEntry(string filePath, EntryType type)
         {
             FilePath = filePath;
             Type = type;
-            
+
+        }
+
+        public void AddReference(ISymbol symbol)
+        {
+            //ToDisplayString formats the symbol as <namespace>.<typeName>
+            //stringifying this helps with the lookup, as we can't use a 
+            //HashSet<ISymbol>::Contains to look them up
+            References.Add(symbol.ToDisplayString());
+        }
+
+        public bool ReferencesSymbol(ISymbol symbol)
+        {
+            //ToDisplayString formats the symbol as <namespace>.<typeName>
+            var synName = symbol.ToDisplayString();
+            return References.Contains(synName);
         }
 
         public void AddDependency(string dependency)
         {
-            if(dependency != FilePath)
+            if (dependency != FilePath)
             {
                 Dependencies.Add(dependency);
             }
         }
 
-        
+
     }
 }

DotNETDepends/Properties/launchSettings.json

@@ -2,7 +2,7 @@
   "profiles": {
     "DotNETDepends": {
       "commandName": "Project",
-      "commandLineArgs": "\"C:\\Users\\Marcus\\projects\\With Space\\NetCoreMVC\\NetCoreMVC.sln\""
+      "commandLineArgs": "\"C:\\Users\\Marcus\\projects\\AspNetWebStack\\Runtime.sln\""
     }
   }
 }

DotNETDepends/RoslynProject.cs

@@ -1,10 +1,5 @@
 using Disassembler;
 using Microsoft.CodeAnalysis;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace DotNETDepends
 {
@@ -14,22 +9,46 @@ namespace DotNETDepends
     internal class SymbolDefinitionFinder : SyntaxWalker
     {
         private readonly DependencyEntry entry;
-        public SymbolDefinitionFinder(DependencyEntry entry) : base(SyntaxWalkerDepth.Node)
+        private readonly SemanticModel semanticModel;
+        public SymbolDefinitionFinder(DependencyEntry entry, SemanticModel semanticModel) : base(SyntaxWalkerDepth.Node)
         {
             this.entry = entry;
+            this.semanticModel = semanticModel; 
         }
 
         public override void Visit(SyntaxNode? node)
         {
             if (node != null)
             {
-                var symbol = entry.Semantic?.GetDeclaredSymbol(node);
-                //Look for NamedTypes.  These are classes.
+                var symbol = semanticModel.GetDeclaredSymbol(node);
+
+                //Look for NamedTypes.  These are classes defined in the file
                 if (symbol != null && symbol.Kind == SymbolKind.NamedType)
                 {
                     entry.Symbols.Add(symbol);
+
+                }
+                else
+                {
+                    //Look for references
+                    var info = semanticModel.GetSymbolInfo(node);
+                    
+                    if (info.Symbol != null)
+                    {
+                        var sym = info.Symbol;
+                        if (sym.Kind != SymbolKind.NamedType)
+                        {
+                            var type = sym.ContainingType;
+                            if (type != null)
+                            {
+                                entry.AddReference(type);
+                            }
+                        }
+                        else { entry.AddReference(sym); }
+
+                    }
+
                 }
-                //We can optimize this in the future to not descend in to uninteresting nodes
                 base.Visit(node);
             }
         }
@@ -56,18 +75,16 @@ public async Task Analyze()
             var compilation = await project.GetCompilationAsync().ConfigureAwait(false);
             if (compilation != null && solutionRoot != null)
             {
-
                 foreach (var tree in compilation.SyntaxTrees)
                 {
-
                     //Make sure we don't add generated files that can get pulled in.
                     if (tree.FilePath.StartsWith(solutionRoot))
                     {
                         //Register the file/type
-                        var fileDep = dependencies.CreateCodeFileEntry(Path.GetRelativePath(solutionRoot, tree.FilePath), compilation.GetSemanticModel(tree), tree);
-                        var walker = new SymbolDefinitionFinder(fileDep);
+                        var fileDep = dependencies.CreateCodeFileEntry(Path.GetRelativePath(solutionRoot, tree.FilePath));
+                        var walker = new SymbolDefinitionFinder(fileDep, compilation.GetSemanticModel(tree));
                         //walk the syntaxtree to find referencable symbols
-                        walker.Visit(fileDep.Tree?.GetRoot());
+                        walker.Visit(tree.GetRoot());
                     }
                 }
             }

DotNETDepends/SolutionReader.cs

@@ -1,7 +1,5 @@
 using Microsoft.CodeAnalysis.MSBuild;
 using Microsoft.CodeAnalysis;
-using Microsoft.CodeAnalysis.CSharp;
-using Microsoft.CodeAnalysis.FindSymbols;
 using System.Diagnostics;
 using System.Runtime.InteropServices;
 using Disassembler;
@@ -38,24 +36,74 @@ public async Task ReadSolutionAsync(String path, AnalysisOutput output)
             Console.WriteLine("Reading solution: " + path);
             var workspace = MSBuildWorkspace.Create();
             var solution = await workspace.OpenSolutionAsync(path).ConfigureAwait(false);
-            
+
             RestoreSolution(solution, output);
 
             var depGraph = solution.GetProjectDependencyGraph();
+
+            CreateProjectDependencies(solution, depGraph);
+
             var dependencyOrderedProjects = depGraph.GetTopologicallySortedProjects();
-            //Read all of the projects declared types and references (if ASP.NET)
+
+            foreach (var projectId in dependencyOrderedProjects)
+            {
+                var project = solution.GetProject(projectId);
+                if (project != null)
+                {
+                    await project.GetCompilationAsync().ConfigureAwait(false);
+                }
+            }
             foreach (var projectId in dependencyOrderedProjects)
             {
-                await ProcessProjectAsync(projectId, solution, depGraph, output).ConfigureAwait(false);
+                await ProcessProjectAsync(projectId, solution, output).ConfigureAwait(false);
+                //once we process the project, remove it so that all of the compilation, ASTs, etc
+                //can be garbage collected.  This results in a new solution
+                solution = solution.RemoveProject(projectId);
             }
-            await ResolveDepenedenciesAsync(solution).ConfigureAwait(false);
+
+
+            ResolveDependencies(solution);
             dependencies.GetLinks(output);
         }
 
         /**
-         * Processes all of the declared and referenced symbols we collected
+         * Creates the dependency graph of the solution and all it's projects.  Has to be done
+         * before we start removing projects from the solution
          */
-        private async Task ResolveDepenedenciesAsync(Solution solution)
+        private void CreateProjectDependencies(Solution solution, ProjectDependencyGraph depGraph)
+        {
+            var solutionDir = Path.GetDirectoryName(solution.FilePath);
+
+            if (solution.FilePath != null && solutionDir != null)
+            {
+                DependencyEntry solutionEntry = dependencies.CreateProjectEntry(Path.GetFileName(solution.FilePath));
+
+                foreach (var project in solution.Projects)
+                {
+                    if (project.FilePath != null)
+                    {
+                        var projectRelativePath = Path.GetRelativePath(solutionDir, project.FilePath);
+                        solutionEntry.AddDependency(projectRelativePath);
+                        DependencyEntry projectEntry = dependencies.CreateProjectEntry(projectRelativePath);
+                        var depProjects = depGraph.GetProjectsThatDirectlyDependOnThisProject(project.Id);
+                        foreach (var depProject in depProjects)
+                        {
+                            var resolved = solution.GetProject(depProject);
+                            if (resolved != null && resolved.FilePath != null)
+                            {
+                                projectEntry.AddDependency(Path.GetRelativePath(solutionDir, resolved.FilePath));
+                            }
+                        }
+                    }
+
+                }
+            }
+        }
+
+        /**
+        * Processes all of the declared and referenced symbols we collected
+        */
+        private void ResolveDependencies(Solution solution)
         {
             var solutionRoot = Path.GetDirectoryName(solution.FilePath);
             if (solutionRoot != null)
@@ -64,7 +112,9 @@ private async Task ResolveDepenedenciesAsync(Solution solution)
                  * These entries come from Roslyn.  They are the declared types in
                  * each file we walked.
                  */
-                foreach (var entry in dependencies.GetFileEntries())
+                var fileEntries = dependencies.GetFileEntries();
+                Console.WriteLine("Resolving references for " + fileEntries.Count + " files.");
+                foreach (var entry in fileEntries)
                 {
 
                     foreach (var symbol in entry.Symbols)
@@ -73,23 +123,17 @@ private async Task ResolveDepenedenciesAsync(Solution solution)
                          * For every symbol declaration we found, find all references in the solution to that symbol.
                          * This will only cover C# abd VB source files.
                          */
-                        var references = await SymbolFinder.FindReferencesAsync(symbol, solution).ConfigureAwait(false);
-
-                        foreach (var reference in references)
+                        foreach (var otherEntry in dependencies.GetFileEntries())
                         {
-                            foreach (var location in reference.Locations)
+                            if (otherEntry != entry)
                             {
-                                //CandidateLocations are guesses by Roslyn.
-                                //Also filter anything we don't have source for
-                                if (location.Location.IsInSource)
+                                if (otherEntry.ReferencesSymbol(symbol))
                                 {
-                                    //Record the reference
-                                    var path = location.Location.SourceTree.FilePath;
-                                    var sourceEntry = dependencies.GetEntry(Path.GetRelativePath(solutionRoot, path));
-                                    sourceEntry?.AddDependency(entry.FilePath);
+                                    otherEntry.AddDependency(entry.FilePath);
                                 }
                             }
                         }
+
                         /**
                          * Now look for references in the ASP.NET file we disassembled.
                          */
@@ -100,6 +144,7 @@ private async Task ResolveDepenedenciesAsync(Solution solution)
                         }
 
                     }
+                    entry.Symbols.Clear();
                 }
                 /**
                  * Find dependencies within all of the decompiled source types
@@ -117,11 +162,12 @@ private async Task ResolveDepenedenciesAsync(Solution solution)
             }
         }
 
+
         /**
          * Runs dotnet restore on the solution.  This fetches any nuget dependencies
          * so that when we compile with Roslyn or decompile the assembly they are available.
          */
-        private bool RestoreSolution(Solution solution, AnalysisOutput analysisOutput)
+        private static bool RestoreSolution(Solution solution, AnalysisOutput analysisOutput)
         {
             var dotnetPath = SDKTools.GetDotnetPath();
             if (solution.FilePath != null)
@@ -245,7 +291,7 @@ private static bool ProjectContainsNETWebFiles(Project project)
          * It then uses Roslyn to parse the references of any C# or VB files, regardless of
          * whether or not it is an ASP.NET project.
          */
-        private async Task ProcessProjectAsync(ProjectId projectId, Solution solution, ProjectDependencyGraph depGraph, AnalysisOutput analysisOutput)
+        private async Task ProcessProjectAsync(ProjectId projectId, Solution solution, AnalysisOutput analysisOutput)
         {
 
             var project = solution.GetProject(projectId);
@@ -254,19 +300,7 @@ private async Task ProcessProjectAsync(ProjectId projectId, Solution solution, P
             if (project != null && project.FilePath != null && solutionRoot != null)
             {
                 Console.WriteLine("Processing project: " + project.FilePath);
-                var depEntry = dependencies.CreateProjectEntry(Path.GetRelativePath(solutionRoot, project.FilePath));
 
-                var projectDependencies = depGraph.GetProjectsThatThisProjectDirectlyDependsOn(projectId);
-                //Add all dependent projects to the entry
-                foreach (var depId in projectDependencies)
-                {
-                    var depProject = solution.GetProject(depId);
-                    if (depProject != null && depProject.FilePath != null)
-                    {
-                        depEntry.AddDependency(depProject.FilePath);
-                    }
-                }
-                
                 //Check for ASP.NET files
                 if (ProjectContainsNETWebFiles(project))
                 {
@@ -296,6 +330,7 @@ private async Task ProcessProjectAsync(ProjectId projectId, Solution solution, P
 
                 }
             }
+
         }
     }
 }