Merge pull request #15070 from tamasvajk/standalone/exclusions

C#: Remove unneeded options and add support for `paths/paths-ignore` in standalone

Author: tamasvajk

csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs

@@ -590,15 +590,15 @@ public void TestLinuxBuildlessExtractionFailed()
         [Fact]
         public void TestLinuxBuildlessExtractionSolution()
         {
-            actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone foo.sln"] = 0;
+            actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 0;
             actions.FileExists["csharp.log"] = true;
             actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
             actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
             actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
             actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
             actions.EnumerateDirectories[@"C:\Project"] = "";
 
-            var autobuilder = CreateAutoBuilder(false, buildless: "true", solution: "foo.sln");
+            var autobuilder = CreateAutoBuilder(false, buildless: "true");
             TestAutobuilderScript(autobuilder, 0, 1);
         }
 
@@ -890,15 +890,15 @@ public void TestSkipNugetMsBuild()
         [Fact]
         public void TestSkipNugetBuildless()
         {
-            actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone foo.sln --skip-nuget"] = 0;
+            actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 0;
             actions.FileExists["csharp.log"] = true;
             actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
             actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
             actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
             actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
             actions.EnumerateDirectories[@"C:\Project"] = "";
 
-            var autobuilder = CreateAutoBuilder(false, buildless: "true", solution: "foo.sln", nugetRestore: "false");
+            var autobuilder = CreateAutoBuilder(false, buildless: "true");
             TestAutobuilderScript(autobuilder, 0, 1);
         }
 

csharp/autobuilder/Semmle.Autobuild.CSharp/StandaloneBuildRule.cs

@@ -17,7 +17,7 @@ internal StandaloneBuildRule(string? dotNetPath)
 
         public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool auto)
         {
-            BuildScript GetCommand(string? solution)
+            BuildScript GetCommand()
             {
                 string standalone;
                 if (builder.CodeQLExtractorLangRoot is not null && builder.CodeQlPlatform is not null)
@@ -32,14 +32,6 @@ BuildScript GetCommand(string? solution)
                 var cmd = new CommandBuilder(builder.Actions);
                 cmd.RunCommand(standalone);
 
-                if (solution is not null)
-                    cmd.QuoteArgument(solution);
-
-                if (!builder.Options.NugetRestore)
-                {
-                    cmd.Argument("--skip-nuget");
-                }
-
                 if (!string.IsNullOrEmpty(this.dotNetPath))
                 {
                     cmd.Argument("--dotnet");
@@ -50,16 +42,11 @@ BuildScript GetCommand(string? solution)
             }
 
             if (!builder.Options.Buildless)
+            {
                 return BuildScript.Failure;
+            }
 
-            if (!builder.Options.Solution.Any())
-                return GetCommand(null);
-
-            var script = BuildScript.Success;
-            foreach (var solution in builder.Options.Solution)
-                script &= GetCommand(solution);
-
-            return script;
+            return GetCommand();
         }
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -80,67 +80,16 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
             this.nonGeneratedSources = allNonBinaryFiles.SelectFileNamesByExtension(".cs").ToList();
             this.generatedSources = new();
             var allProjects = allNonBinaryFiles.SelectFileNamesByExtension(".csproj");
-            var solutions = options.SolutionFile is not null
-                ? new[] { options.SolutionFile }
-                : allNonBinaryFiles.SelectFileNamesByExtension(".sln");
-            var dllPaths = options.DllDirs.Count == 0
-                ? allFiles.SelectFileNamesByExtension(".dll").ToHashSet()
-                : options.DllDirs.Select(Path.GetFullPath).ToHashSet();
-
-            if (options.UseNuGet)
-            {
-                try
-                {
-                    var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, progressMonitor);
-                    nuget.InstallPackages();
-
-                    var nugetPackageDlls = legacyPackageDirectory.DirInfo.GetFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = true });
-                    var nugetPackageDllPaths = nugetPackageDlls.Select(f => f.FullName).ToHashSet();
-                    var excludedPaths = nugetPackageDllPaths
-                        .Where(path => IsPathInSubfolder(path, legacyPackageDirectory.DirInfo.FullName, "tools"));
-
-                    foreach (var excludedPath in excludedPaths)
-                    {
-                        progressMonitor.LogInfo($"Excluded Nuget DLL: {excludedPath}");
-                    }
-
-                    nugetPackageDllPaths.ExceptWith(excludedPaths);
-                    dllPaths.UnionWith(nugetPackageDllPaths);
-                }
-                catch (FileNotFoundException)
-                {
-                    progressMonitor.MissingNuGet();
-                }
-
-                var restoredProjects = RestoreSolutions(solutions, out var assets1);
-                var projects = allProjects.Except(restoredProjects);
-                RestoreProjects(projects, out var assets2);
-
-                var dependencies = Assets.GetCompilationDependencies(progressMonitor, assets1.Union(assets2));
-
-                var paths = dependencies
-                    .Paths
-                    .Select(d => Path.Combine(packageDirectory.DirInfo.FullName, d))
-                    .ToList();
-                dllPaths.UnionWith(paths);
-
-                LogAllUnusedPackages(dependencies);
-                DownloadMissingPackages(allNonBinaryFiles, dllPaths);
-            }
-
-            var frameworkLocations = new HashSet<string>();
+            var allSolutions = allNonBinaryFiles.SelectFileNamesByExtension(".sln");
+            var dllPaths = allFiles.SelectFileNamesByExtension(".dll").ToHashSet();
 
+            RestoreNugetPackages(allNonBinaryFiles, allProjects, allSolutions, dllPaths);
             // Find DLLs in the .Net / Asp.Net Framework
-            // This block needs to come after the nuget restore, because the nuget restore might fetch the .NET Core/Framework reference assemblies.
-            if (options.ScanNetFrameworkDlls)
-            {
-                AddNetFrameworkDlls(dllPaths, frameworkLocations);
-                AddAspNetCoreFrameworkDlls(dllPaths, frameworkLocations);
-                AddMicrosoftWindowsDesktopDlls(dllPaths, frameworkLocations);
-            }
+            // This needs to come after the nuget restore, because the nuget restore might fetch the .NET Core/Framework reference assemblies.
+            var frameworkLocations = AddFrameworkDlls(dllPaths);
 
             assemblyCache = new AssemblyCache(dllPaths, frameworkLocations, progressMonitor);
-            AnalyseSolutions(solutions);
+            AnalyseSolutions(allSolutions);
 
             foreach (var filename in assemblyCache.AllAssemblies.Select(a => a.Filename))
             {
@@ -182,6 +131,58 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
                 DateTime.Now - startTime);
         }
 
+        private HashSet<string> AddFrameworkDlls(HashSet<string> dllPaths)
+        {
+            var frameworkLocations = new HashSet<string>();
+
+            AddNetFrameworkDlls(dllPaths, frameworkLocations);
+            AddAspNetCoreFrameworkDlls(dllPaths, frameworkLocations);
+            AddMicrosoftWindowsDesktopDlls(dllPaths, frameworkLocations);
+
+            return frameworkLocations;
+        }
+
+        private void RestoreNugetPackages(List<FileInfo> allNonBinaryFiles, IEnumerable<string> allProjects, IEnumerable<string> allSolutions, HashSet<string> dllPaths)
+        {
+            try
+            {
+                var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, progressMonitor);
+                nuget.InstallPackages();
+
+                var nugetPackageDlls = legacyPackageDirectory.DirInfo.GetFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = true });
+                var nugetPackageDllPaths = nugetPackageDlls.Select(f => f.FullName).ToHashSet();
+                var excludedPaths = nugetPackageDllPaths
+                    .Where(path => IsPathInSubfolder(path, legacyPackageDirectory.DirInfo.FullName, "tools"));
+
+                foreach (var excludedPath in excludedPaths)
+                {
+                    progressMonitor.LogInfo($"Excluded Nuget DLL: {excludedPath}");
+                }
+
+                nugetPackageDllPaths.ExceptWith(excludedPaths);
+                dllPaths.UnionWith(nugetPackageDllPaths);
+            }
+            catch (FileNotFoundException)
+            {
+                progressMonitor.MissingNuGet();
+            }
+
+            var restoredProjects = RestoreSolutions(allSolutions, out var assets1);
+            var projects = allProjects.Except(restoredProjects);
+            RestoreProjects(projects, out var assets2);
+
+            var dependencies = Assets.GetCompilationDependencies(progressMonitor, assets1.Union(assets2));
+
+            var paths = dependencies
+                .Paths
+                .Select(d => Path.Combine(packageDirectory.DirInfo.FullName, d))
+                .ToList();
+            dllPaths.UnionWith(paths);
+
+            LogAllUnusedPackages(dependencies);
+            DownloadMissingPackages(allNonBinaryFiles, dllPaths);
+        }
+
         private static bool IsPathInSubfolder(string path, string rootFolder, string subFolder)
         {
             return path.IndexOf(
@@ -192,11 +193,6 @@ private static bool IsPathInSubfolder(string path, string rootFolder, string sub
 
         private void RemoveNugetAnalyzerReferences()
         {
-            if (!options.UseNuGet)
-            {
-                return;
-            }
-
             var packageFolder = packageDirectory.DirInfo.FullName.ToLowerInvariant();
             if (packageFolder == null)
             {
@@ -279,11 +275,7 @@ private void AddNetFrameworkDlls(ISet<string> dllPaths, ISet<string> frameworkLo
 
             string? runtimeLocation = null;
 
-            if (options.UseSelfContainedDotnet)
-            {
-                runtimeLocation = Runtime.ExecutingRuntime;
-            }
-            else if (fileContent.IsNewProjectStructureUsed)
+            if (fileContent.IsNewProjectStructureUsed)
             {
                 runtimeLocation = Runtime.NetCoreRuntime;
             }
@@ -301,11 +293,6 @@ private void AddNetFrameworkDlls(ISet<string> dllPaths, ISet<string> frameworkLo
 
         private void RemoveNugetPackageReference(string packagePrefix, ISet<string> dllPaths)
         {
-            if (!options.UseNuGet)
-            {
-                return;
-            }
-
             var packageFolder = packageDirectory.DirInfo.FullName.ToLowerInvariant();
             if (packageFolder == null)
             {
@@ -353,11 +340,6 @@ private void AddMicrosoftWindowsDesktopDlls(ISet<string> dllPaths, ISet<string>
 
         private string? GetPackageDirectory(string packagePrefix)
         {
-            if (!options.UseNuGet)
-            {
-                return null;
-            }
-
             return new DirectoryInfo(packageDirectory.DirInfo.FullName)
                 .EnumerateDirectories(packagePrefix + "*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
                 .FirstOrDefault()?
@@ -366,11 +348,6 @@ private void AddMicrosoftWindowsDesktopDlls(ISet<string> dllPaths, ISet<string>
 
         private IEnumerable<string> GetAllPackageDirectories()
         {
-            if (!options.UseNuGet)
-            {
-                return Enumerable.Empty<string>();
-            }
-
             return new DirectoryInfo(packageDirectory.DirInfo.FullName)
                 .EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
                 .Select(d => d.Name);
@@ -455,14 +432,14 @@ private void GenerateSourceFilesFromWebViews(List<FileInfo> allFiles)
 
         private IEnumerable<FileInfo> GetAllFiles()
         {
-            var files = sourceDir.GetFiles("*.*", new EnumerationOptions { RecurseSubdirectories = true })
-                .Where(d => !options.ExcludesFile(d.FullName));
+            IEnumerable<FileInfo> files = sourceDir.GetFiles("*.*", new EnumerationOptions { RecurseSubdirectories = true });
 
             if (options.DotNetPath != null)
             {
                 files = files.Where(f => !f.FullName.StartsWith(options.DotNetPath, StringComparison.OrdinalIgnoreCase));
             }
 
+            files = new FilePathFilter(sourceDir, progressMonitor).Filter(files);
             return files;
         }
 

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyOptions.cs

@@ -10,43 +10,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
     /// </summary>
     public interface IDependencyOptions
     {
-        /// <summary>
-        /// Directories to search DLLs in.
-        /// </summary>
-        IList<string> DllDirs { get; }
-
-        /// <summary>
-        /// Files/patterns to exclude.
-        /// </summary>
-        IList<string> Excludes { get; }
-
-        /// <summary>
-        /// Whether to analyse NuGet packages.
-        /// </summary>
-        bool UseNuGet { get; }
-
-        /// <summary>
-        /// The solution file to analyse, or null if not specified.
-        /// </summary>
-        string? SolutionFile { get; }
-
-        /// <summary>
-        /// Whether to use the packaged dotnet runtime.
-        /// </summary>
-        bool UseSelfContainedDotnet { get; }
-
-        /// <summary>
-        /// Whether to search the .Net framework directory.
-        /// </summary>
-        bool ScanNetFrameworkDlls { get; }
-
-        /// <summary>
-        /// Determine whether the given path should be excluded.
-        /// </summary>
-        /// <param name="path">The path to query.</param>
-        /// <returns>True iff the path matches an exclusion.</returns>
-        bool ExcludesFile(string path);
-
         /// <summary>
         /// The number of threads to use.
         /// </summary>
@@ -62,21 +25,6 @@ public class DependencyOptions : IDependencyOptions
     {
         public static IDependencyOptions Default => new DependencyOptions();
 
-        public IList<string> DllDirs { get; set; } = new List<string>();
-
-        public IList<string> Excludes { get; set; } = new List<string>();
-
-        public bool UseNuGet { get; set; } = true;
-
-        public string? SolutionFile { get; set; }
-
-        public bool UseSelfContainedDotnet { get; set; } = false;
-
-        public bool ScanNetFrameworkDlls { get; set; } = true;
-
-        public bool ExcludesFile(string path) =>
-            Excludes.Any(path.Contains);
-
         public int Threads { get; set; } = EnvironmentVariables.GetDefaultNumberOfThreads();
 
         public string? DotNetPath { get; set; } = null;