C#: Fetch file info fewer times and make dependencies more clear.

Author: michaelnebel

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -59,13 +59,13 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
             this.progressMonitor.FindingFiles(srcDir);
 
             packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName));
-
-            this.fileContent = new FileContent(progressMonitor, () => GetFiles("*.*"));
-            this.allSources = GetFiles("*.cs").ToList();
-            var allProjects = GetFiles("*.csproj");
+            var allFiles = GetFiles("*.*").ToList();
+            this.fileContent = new FileContent(progressMonitor, GetFileNames(allFiles));
+            this.allSources = GetFileNames(allFiles, ".cs").ToList();
+            var allProjects = GetFileNames(allFiles, ".csproj");
             var solutions = options.SolutionFile is not null
                 ? new[] { options.SolutionFile }
-                : GetFiles("*.sln");
+                : GetFileNames(allFiles, ".sln");
 
             var dllDirNames = options.DllDirs.Select(Path.GetFullPath).ToList();
 
@@ -107,7 +107,7 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
                 {
                     Restore(solutions);
                     Restore(allProjects);
-                    DownloadMissingPackages();
+                    DownloadMissingPackages(allFiles);
                 }
             }
 
@@ -136,7 +136,7 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
             if (bool.TryParse(webViewExtractionOption, out var shouldExtractWebViews) &&
                 shouldExtractWebViews)
             {
-                GenerateSourceFilesFromWebViews();
+                GenerateSourceFilesFromWebViews(allFiles);
             }
 
             progressMonitor.Summary(
@@ -151,13 +151,11 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
                 DateTime.Now - startTime);
         }
 
-        private void GenerateSourceFilesFromWebViews()
+        private void GenerateSourceFilesFromWebViews(List<FileInfo> allFiles)
         {
             progressMonitor.LogInfo($"Generating source files from cshtml and razor files.");
 
-            var views = GetFiles("*.cshtml")
-                .Concat(GetFiles("*.razor"))
-                .ToArray();
+            var views = GetFileNames(allFiles, ".cshtml", ".razor").ToArray();
 
             if (views.Length > 0)
             {
@@ -185,15 +183,16 @@ private void GenerateSourceFilesFromWebViews()
 
         public DependencyManager(string srcDir) : this(srcDir, DependencyOptions.Default, new ConsoleLogger(Verbosity.Info)) { }
 
-        private IEnumerable<string> GetFiles(string pattern, bool recurseSubdirectories = true) =>
+        private IEnumerable<FileInfo> GetFiles(string pattern, bool recurseSubdirectories = true) =>
              sourceDir.GetFiles(pattern, new EnumerationOptions
              {
                  RecurseSubdirectories = recurseSubdirectories,
                  MatchCasing = MatchCasing.CaseInsensitive
              })
-                .Where(d => d.Extension != ".dll")
-                .Select(d => d.FullName)
-                .Where(d => !options.ExcludesFile(d));
+                .Where(d => d.Extension != ".dll" && !options.ExcludesFile(d.FullName));
+
+        private static IEnumerable<string> GetFileNames(IEnumerable<FileInfo> files, params string[] extensions) =>
+            files.Where(fi => !extensions.Any() || extensions.Contains(fi.Extension)).Select(fi => fi.FullName);
 
         /// <summary>
         /// Computes a unique temp directory for the packages associated
@@ -374,14 +373,19 @@ private void Restore(IEnumerable<string> targets, string? pathToNugetConfig = nu
             }
         }
 
-        private void DownloadMissingPackages()
+        private void DownloadMissingPackages(List<FileInfo> allFiles)
         {
-            var nugetConfigs = GetFiles("nuget.config", recurseSubdirectories: true).ToArray();
+            var nugetConfigs = allFiles
+                .Where(fi => fi.Name == "nuget.config")
+                .Select(fi => fi.FullName)
+                .ToArray();
             string? nugetConfig = null;
             if (nugetConfigs.Length > 1)
             {
                 progressMonitor.MultipleNugetConfig(nugetConfigs);
-                nugetConfig = GetFiles("nuget.config", recurseSubdirectories: false).FirstOrDefault();
+                nugetConfig = GetFiles("nuget.config", recurseSubdirectories: false)
+                    .Select(fi => fi.FullName)
+                    .FirstOrDefault();
                 if (nugetConfig == null)
                 {
                     progressMonitor.NoTopLevelNugetConfig();

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs

@@ -1,7 +1,6 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
-using System.Linq;
 using System.Text.RegularExpressions;
 using Semmle.Util;
 
@@ -18,7 +17,7 @@ internal partial class FileContent
     {
         private readonly ProgressMonitor progressMonitor;
         private readonly IUnsafeFileReader unsafeFileReader;
-        private readonly Func<IEnumerable<string>> getFiles;
+        private readonly IEnumerable<string> files;
         private readonly HashSet<string> allPackages = new HashSet<string>();
         private readonly Initializer initialize;
 
@@ -50,17 +49,17 @@ public bool UseAspNetDlls
         }
 
         internal FileContent(ProgressMonitor progressMonitor,
-            Func<IEnumerable<string>> getFiles,
+            IEnumerable<string> files,
             IUnsafeFileReader unsafeFileReader)
         {
             this.progressMonitor = progressMonitor;
-            this.getFiles = getFiles;
+            this.files = files;
             this.unsafeFileReader = unsafeFileReader;
             this.initialize = new Initializer(DoInitialize);
         }
 
 
-        public FileContent(ProgressMonitor progressMonitor, Func<IEnumerable<string>> getFiles) : this(progressMonitor, getFiles, new UnsafeFileReader())
+        public FileContent(ProgressMonitor progressMonitor, IEnumerable<string> files) : this(progressMonitor, files, new UnsafeFileReader())
         { }
 
         private static string GetGroup(ReadOnlySpan<char> input, ValueMatch valueMatch, string groupPrefix)
@@ -95,7 +94,7 @@ private static bool IsGroupMatch(ReadOnlySpan<char> line, Regex regex, string gr
 
         private void DoInitialize()
         {
-            foreach (var file in getFiles())
+            foreach (var file in files)
             {
                 try
                 {

csharp/extractor/Semmle.Extraction.Tests/FileContent.cs

@@ -35,7 +35,7 @@ public IEnumerable<string> ReadLines(string file)
     internal class TestFileContent : FileContent
     {
         public TestFileContent(List<string> lines) : base(new ProgressMonitor(new LoggerStub()),
-            () => new List<string>() { "test1.cs" },
+            new List<string>() { "test1.cs" },
             new UnsafeFileReaderStub(lines))
         { }
     }