Skip to content

Commit e7dbe9f

Browse files
michaelnebelmichaelnebel
authored
Merge pull request #14028 from michaelnebel/csharp/dependencygetfiles
C#: Improve GetFiles in the Dependency Manager.
2 parents 38b7812 + 02b8adf commit e7dbe9f

File tree

5 files changed

+66
-33
lines changed

5 files changed

+66
-33
lines changed

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs‎

Lines changed: 20 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -59,13 +59,14 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
5959
this.progressMonitor.FindingFiles(srcDir);
6060

6161
packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName));
62-
63-
this.fileContent = new FileContent(progressMonitor, () => GetFiles("*.*"));
64-
this.allSources = GetFiles("*.cs").ToList();
65-
var allProjects = GetFiles("*.csproj");
62+
var allFiles = GetAllFiles().ToList();
63+
var smallFiles = allFiles.SelectSmallFiles(progressMonitor).SelectFileNames();
64+
this.fileContent = new FileContent(progressMonitor, smallFiles);
65+
this.allSources = allFiles.SelectFileNamesByExtension(".cs").ToList();
66+
var allProjects = allFiles.SelectFileNamesByExtension(".csproj");
6667
var solutions = options.SolutionFile is not null
6768
? new[] { options.SolutionFile }
68-
: GetFiles("*.sln");
69+
: allFiles.SelectFileNamesByExtension(".sln");
6970

7071
var dllDirNames = options.DllDirs.Select(Path.GetFullPath).ToList();
7172

@@ -107,7 +108,7 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
107108
{
108109
Restore(solutions);
109110
Restore(allProjects);
110-
DownloadMissingPackages();
111+
DownloadMissingPackages(allFiles);
111112
}
112113
}
113114

@@ -136,7 +137,7 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
136137
if (bool.TryParse(webViewExtractionOption, out var shouldExtractWebViews) &&
137138
shouldExtractWebViews)
138139
{
139-
GenerateSourceFilesFromWebViews();
140+
GenerateSourceFilesFromWebViews(allFiles);
140141
}
141142

142143
progressMonitor.Summary(
@@ -151,13 +152,11 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
151152
DateTime.Now - startTime);
152153
}
153154

154-
private void GenerateSourceFilesFromWebViews()
155+
private void GenerateSourceFilesFromWebViews(List<FileInfo> allFiles)
155156
{
156157
progressMonitor.LogInfo($"Generating source files from cshtml and razor files.");
157158

158-
var views = GetFiles("*.cshtml")
159-
.Concat(GetFiles("*.razor"))
160-
.ToArray();
159+
var views = allFiles.SelectFileNamesByExtension(".cshtml", ".razor").ToArray();
161160

162161
if (views.Length > 0)
163162
{
@@ -185,15 +184,9 @@ private void GenerateSourceFilesFromWebViews()
185184

186185
public DependencyManager(string srcDir) : this(srcDir, DependencyOptions.Default, new ConsoleLogger(Verbosity.Info)) { }
187186

188-
private IEnumerable<string> GetFiles(string pattern, bool recurseSubdirectories = true) =>
189-
sourceDir.GetFiles(pattern, new EnumerationOptions
190-
{
191-
RecurseSubdirectories = recurseSubdirectories,
192-
MatchCasing = MatchCasing.CaseInsensitive
193-
})
194-
.Where(d => d.Extension != ".dll")
195-
.Select(d => d.FullName)
196-
.Where(d => !options.ExcludesFile(d));
187+
private IEnumerable<FileInfo> GetAllFiles() =>
188+
sourceDir.GetFiles("*.*", new EnumerationOptions { RecurseSubdirectories = true })
189+
.Where(d => d.Extension != ".dll" && !options.ExcludesFile(d.FullName));
197190

198191
/// <summary>
199192
/// Computes a unique temp directory for the packages associated
@@ -374,14 +367,17 @@ private void Restore(IEnumerable<string> targets, string? pathToNugetConfig = nu
374367
}
375368
}
376369

377-
private void DownloadMissingPackages()
370+
private void DownloadMissingPackages(List<FileInfo> allFiles)
378371
{
379-
var nugetConfigs = GetFiles("nuget.config", recurseSubdirectories: true).ToArray();
372+
var nugetConfigs = allFiles.SelectFileNamesByName("nuget.config").ToArray();
380373
string? nugetConfig = null;
381374
if (nugetConfigs.Length > 1)
382375
{
383376
progressMonitor.MultipleNugetConfig(nugetConfigs);
384-
nugetConfig = GetFiles("nuget.config", recurseSubdirectories: false).FirstOrDefault();
377+
nugetConfig = allFiles
378+
.SelectRootFiles(sourceDir)
379+
.SelectFileNamesByName("nuget.config")
380+
.FirstOrDefault();
385381
if (nugetConfig == null)
386382
{
387383
progressMonitor.NoTopLevelNugetConfig();
@@ -393,8 +389,7 @@ private void DownloadMissingPackages()
393389
}
394390

395391
var alreadyDownloadedPackages = Directory.GetDirectories(packageDirectory.DirInfo.FullName)
396-
.Select(d => Path.GetFileName(d)
397-
.ToLowerInvariant());
392+
.Select(d => Path.GetFileName(d).ToLowerInvariant());
398393
var notYetDownloadedPackages = fileContent.AllPackages.Except(alreadyDownloadedPackages);
399394
foreach (var package in notYetDownloadedPackages)
400395
{

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
using System;
22
using System.Collections.Generic;
33
using System.IO;
4-
using System.Linq;
54
using System.Text.RegularExpressions;
65
using Semmle.Util;
76

@@ -18,7 +17,7 @@ internal partial class FileContent
1817
{
1918
private readonly ProgressMonitor progressMonitor;
2019
private readonly IUnsafeFileReader unsafeFileReader;
21-
private readonly Func<IEnumerable<string>> getFiles;
20+
private readonly IEnumerable<string> files;
2221
private readonly HashSet<string> allPackages = new HashSet<string>();
2322
private readonly Initializer initialize;
2423

@@ -50,17 +49,17 @@ public bool UseAspNetDlls
5049
}
5150

5251
internal FileContent(ProgressMonitor progressMonitor,
53-
Func<IEnumerable<string>> getFiles,
52+
IEnumerable<string> files,
5453
IUnsafeFileReader unsafeFileReader)
5554
{
5655
this.progressMonitor = progressMonitor;
57-
this.getFiles = getFiles;
56+
this.files = files;
5857
this.unsafeFileReader = unsafeFileReader;
5958
this.initialize = new Initializer(DoInitialize);
6059
}
6160

6261

63-
public FileContent(ProgressMonitor progressMonitor, Func<IEnumerable<string>> getFiles) : this(progressMonitor, getFiles, new UnsafeFileReader())
62+
public FileContent(ProgressMonitor progressMonitor, IEnumerable<string> files) : this(progressMonitor, files, new UnsafeFileReader())
6463
{ }
6564

6665
private static string GetGroup(ReadOnlySpan<char> input, ValueMatch valueMatch, string groupPrefix)
@@ -95,7 +94,7 @@ private static bool IsGroupMatch(ReadOnlySpan<char> line, Regex regex, string gr
9594

9695
private void DoInitialize()
9796
{
98-
foreach (var file in getFiles())
97+
foreach (var file in files)
9998
{
10099
try
101100
{

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileInfoExtensions.cs‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.IO;
4+
using System.Linq;
5+
6+
namespace Semmle.Extraction.CSharp.DependencyFetching
7+
{
8+
public static class FileInfoExtensions
9+
{
10+
private static IEnumerable<string> SelectFilesAux(this IEnumerable<FileInfo> files, Predicate<FileInfo> p) =>
11+
files.Where(f => p(f)).Select(fi => fi.FullName);
12+
13+
public static IEnumerable<FileInfo> SelectRootFiles(this IEnumerable<FileInfo> files, DirectoryInfo dir) =>
14+
files.Where(file => file.DirectoryName == dir.FullName);
15+
16+
internal static IEnumerable<FileInfo> SelectSmallFiles(this IEnumerable<FileInfo> files, ProgressMonitor progressMonitor)
17+
{
18+
const int oneMb = 1_048_576;
19+
return files.Where(file =>
20+
{
21+
if (file.Length > oneMb)
22+
{
23+
progressMonitor.LogDebug($"Skipping {file.FullName} because it is bigger than 1MB.");
24+
return false;
25+
}
26+
return true;
27+
});
28+
}
29+
30+
public static IEnumerable<string> SelectFileNamesByExtension(this IEnumerable<FileInfo> files, params string[] extensions) =>
31+
files.SelectFilesAux(fi => extensions.Contains(fi.Extension));
32+
33+
public static IEnumerable<string> SelectFileNamesByName(this IEnumerable<FileInfo> files, params string[] names) =>
34+
files.SelectFilesAux(fi => names.Any(name => string.Compare(name, fi.Name, true) == 0));
35+
36+
public static IEnumerable<string> SelectFileNames(this IEnumerable<FileInfo> files) =>
37+
files.SelectFilesAux(_ => true);
38+
}
39+
}

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/ProgressMonitor.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ public void Log(Severity severity, string message) =>
1818
public void LogInfo(string message) =>
1919
logger.Log(Severity.Info, message);
2020

21-
private void LogDebug(string message) =>
21+
public void LogDebug(string message) =>
2222
logger.Log(Severity.Debug, message);
2323

2424
private void LogError(string message) =>

‎csharp/extractor/Semmle.Extraction.Tests/FileContent.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ public IEnumerable<string> ReadLines(string file)
3535
internal class TestFileContent : FileContent
3636
{
3737
public TestFileContent(List<string> lines) : base(new ProgressMonitor(new LoggerStub()),
38-
() => new List<string>() { "test1.cs" },
38+
new List<string>() { "test1.cs" },
3939
new UnsafeFileReaderStub(lines))
4040
{ }
4141
}

0 commit comments

Comments
 (0)