C#: Recognize .proj files in autobuilder

When determining the target of `msbuild` or `dotnet build`, first look for `.proj`
files, then `.sln` files, and finally `.csproj`/`.vcxproj` files. In all three cases,
choose the project/solution file closest to the root.

Author: hvitved

change-notes/1.19/analysis-csharp.md

@@ -29,3 +29,7 @@
 
 * `getArgument()` on `AccessorCall` has been improved so it now takes tuple assignments into account. For example, the argument for the implicit `value` parameter in the setter of property `P` is `0` in `(P, x) = (0, 1)`. Additionally, the argument for the `value` parameter in compound assignments is now only the expanded value, for example, in `P += 7` the argument is `P + 7` and not `7`.
 * The predicate `isInArgument()` has been added to the `AssignableAccess` class. This holds for expressions that are passed as arguments using `in`.
+
+## Changes to the autobuilder
+
+* When determining the target of `msbuild` or `dotnet build`, first look for `.proj` files, then `.sln` files, and finally `.csproj`/`.vcxproj` files. In all three cases, choose the project/solution file closest to the root.

csharp/autobuilder/Semmle.Autobuild.Tests/BuildScripts.cs

@@ -29,26 +29,32 @@ interface IBuildRule
     public class Autobuilder
     {
         /// <summary>
-        /// Full file paths of files found in the project directory.
+        /// Full file paths of files found in the project directory, as well
+        /// their distance from the project root folder. The list is sorted
+        /// by distance in ascending order.
         /// </summary>
-        public IEnumerable<string> Paths => pathsLazy.Value;
-        readonly Lazy<IEnumerable<string>> pathsLazy;
+        public IEnumerable<(string, int)> Paths => pathsLazy.Value;
+        readonly Lazy<IEnumerable<(string, int)>> pathsLazy;
 
         /// <summary>
         /// Gets a list of paths matching a set of extensions
-        /// (including the ".").
+        /// (including the "."), as well their distance from the project root folder.
+        /// The list is sorted by distance in ascending order.
         /// </summary>
         /// <param name="extensions">The extensions to find.</param>
         /// <returns>The files matching the extension.</returns>
-        public IEnumerable<string> GetExtensions(params string[] extensions) =>
-            Paths.Where(p => extensions.Contains(Path.GetExtension(p)));
+        public IEnumerable<(string, int)> GetExtensions(params string[] extensions) =>
+            Paths.Where(p => extensions.Contains(Path.GetExtension(p.Item1)));
 
         /// <summary>
-        /// Gets all paths matching a particular filename.
+        /// Gets all paths matching a particular filename, as well
+        /// their distance from the project root folder.  The list is sorted
+        /// by distance in ascending order.
         /// </summary>
         /// <param name="name">The filename to find.</param>
         /// <returns>Possibly empty sequence of paths with the given filename.</returns>
-        public IEnumerable<string> GetFilename(string name) => Paths.Where(p => Path.GetFileName(p) == name);
+        public IEnumerable<(string, int)> GetFilename(string name) =>
+            Paths.Where(p => Path.GetFileName(p.Item1) == name);
 
         /// <summary>
         /// Holds if a given path, relative to the root of the source directory
@@ -59,30 +65,30 @@ public IEnumerable<string> GetExtensions(params string[] extensions) =>
         public bool HasRelativePath(string path) => HasPath(Actions.PathCombine(RootDirectory, path));
 
         /// <summary>
-        /// List of solution files to build.
+        /// List of project/solution files to build.
         /// </summary>
-        public IList<ISolution> SolutionsToBuild => solutionsToBuildLazy.Value;
-        readonly Lazy<IList<ISolution>> solutionsToBuildLazy;
+        public IList<IProjectOrSolution> ProjectsOrSolutionsToBuild => projectsOrSolutionsToBuildLazy.Value;
+        readonly Lazy<IList<IProjectOrSolution>> projectsOrSolutionsToBuildLazy;
 
         /// <summary>
         /// Holds if a given path was found.
         /// </summary>
         /// <param name="path">The path of the file.</param>
         /// <returns>True iff the path was found.</returns>
-        public bool HasPath(string path) => Paths.Any(p => path == p);
+        public bool HasPath(string path) => Paths.Any(p => path == p.Item1);
 
-        void FindFiles(string dir, int depth, IList<string> results)
+        void FindFiles(string dir, int depth, int maxDepth, IList<(string, int)> results)
         {
             foreach (var f in Actions.EnumerateFiles(dir))
             {
-                results.Add(f);
+                results.Add((f, depth));
             }
 
-            if (depth > 1)
+            if (depth < maxDepth)
             {
                 foreach (var d in Actions.EnumerateDirectories(dir))
                 {
-                    FindFiles(d, depth - 1, results);
+                    FindFiles(d, depth + 1, maxDepth, results);
                 }
             }
         }
@@ -113,46 +119,75 @@ public Autobuilder(IBuildActions actions, AutobuildOptions options)
             Actions = actions;
             Options = options;
 
-            pathsLazy = new Lazy<IEnumerable<string>>(() =>
+            pathsLazy = new Lazy<IEnumerable<(string, int)>>(() =>
             {
-                var files = new List<string>();
-                FindFiles(options.RootDirectory, options.SearchDepth, files);
-                return files.
-                    OrderBy(s => s.Count(c => c == Path.DirectorySeparatorChar)).
-                    ThenBy(s => Path.GetFileName(s).Length).
-                    ToArray();
+                var files = new List<(string, int)>();
+                FindFiles(options.RootDirectory, 0, options.SearchDepth, files);
+                return files.OrderBy(f => f.Item2).ToArray();
             });
 
-            solutionsToBuildLazy = new Lazy<IList<ISolution>>(() =>
+            projectsOrSolutionsToBuildLazy = new Lazy<IList<IProjectOrSolution>>(() =>
             {
                 if (options.Solution.Any())
                 {
-                    var ret = new List<ISolution>();
+                    var ret = new List<IProjectOrSolution>();
                     foreach (var solution in options.Solution)
                     {
                         if (actions.FileExists(solution))
                             ret.Add(new Solution(this, solution));
                         else
-                            Log(Severity.Error, "The specified solution file {0} was not found", solution);
+                            Log(Severity.Error, $"The specified solution file {solution} was not found");
                     }
                     return ret;
                 }
 
-                var solutions = GetExtensions(".sln").
-                        Select(s => new Solution(this, s)).
-                        Where(s => s.ProjectCount > 0).
-                        OrderByDescending(s => s.ProjectCount).
-                        ThenBy(s => s.Path.Length).
+                bool FindFiles(string extension, Func<string, ProjectOrSolution> create, out IEnumerable<IProjectOrSolution> files)
+                {
+                    var allFiles = GetExtensions(extension).
+                        Select(p => (ProjectOrSolution: create(p.Item1), DistanceFromRoot: p.Item2)).
+                        Where(p => p.ProjectOrSolution.HasLanguage(this.Options.Language)).
                         ToArray();
 
-                foreach (var sln in solutions)
-                {
-                    Log(Severity.Info, $"Found {sln.Path} with {sln.ProjectCount} {this.Options.Language} projects, version {sln.ToolsVersion}, config {string.Join(" ", sln.Configurations.Select(c => c.FullName))}");
+                    if (allFiles.Length == 0)
+                    {
+                        files = null;
+                        return false;
+                    }
+
+                    if (options.AllSolutions)
+                    {
+                        files = allFiles.Select(p => p.ProjectOrSolution);
+                        return true;
+                    }
+
+                    var firstIsClosest = allFiles.Length > 1 && allFiles[0].DistanceFromRoot < allFiles[1].DistanceFromRoot;
+                    if (allFiles.Length == 1 || firstIsClosest)
+                    {
+                        files = allFiles.Select(p => p.ProjectOrSolution).Take(1);
+                        return true;
+                    }
+
+                    var candidates = allFiles.
+                        Where(f => f.DistanceFromRoot == allFiles[0].DistanceFromRoot).
+                        Select(f => f.ProjectOrSolution);
+                    Log(Severity.Info, $"Found multiple '{extension}' files, giving up: {string.Join(", ", candidates)}.");
+                    files = new IProjectOrSolution[0];
+                    return true;
                 }
 
-                return new List<ISolution>(options.AllSolutions ?
-                    solutions :
-                    solutions.Take(1));
+                // First look for `.proj` files
+                if (FindFiles(".proj", f => new Project(this, f), out var ret1))
+                    return ret1.ToList();
+
+                // Then look for `.sln` files
+                if (FindFiles(".sln", f => new Solution(this, f), out var ret2))
+                    return ret2.ToList();
+
+                // Finally look for language specific project files, e.g. `.csproj` files
+                if (FindFiles(this.Options.Language.ProjectExtension, f => new Project(this, f), out var ret3))
+                    return ret3.ToList();
+
+                return new List<IProjectOrSolution>();
             });
 
             SemmleDist = Actions.GetEnvironmentVariable("SEMMLE_DIST");

csharp/autobuilder/Semmle.Autobuild/Autobuilder.cs

@@ -3,6 +3,7 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
+using System.Xml;
 
 namespace Semmle.Autobuild
 {
@@ -96,12 +97,27 @@ public interface IBuildActions
         /// <returns>The combined path.</returns>
         string PathCombine(params string[] parts);
 
+        /// <summary>
+        /// Gets the full path for <paramref name="path"/>, Path.GetFullPath().
+        /// </summary>
+        string GetFullPath(string path);
+
+        /// <summary>
+        /// Gets the directory of <paramref name="path"/>, Path.GetDirectoryName().
+        /// </summary>
+        string GetDirectoryName(string path);
+
         /// <summary>
         /// Writes contents to file, File.WriteAllText().
         /// </summary>
         /// <param name="filename">The filename.</param>
         /// <param name="contents">The text.</param>
         void WriteAllText(string filename, string contents);
+
+        /// <summary>
+        /// Loads the XML document from <paramref name="filename"/>.
+        /// </summary>
+        XmlDocument LoadXml(string filename);
     }
 
     /// <summary>
@@ -167,10 +183,17 @@ int IBuildActions.RunProcess(string cmd, string args, string workingDirectory, I
 
         void IBuildActions.WriteAllText(string filename, string contents) => File.WriteAllText(filename, contents);
 
-        private SystemBuildActions()
+        XmlDocument IBuildActions.LoadXml(string filename)
         {
+            var ret = new XmlDocument();
+            ret.Load(filename);
+            return ret;
         }
 
+        string IBuildActions.GetFullPath(string path) => Path.GetFullPath(path);
+
+        string IBuildActions.GetDirectoryName(string path) => Path.GetDirectoryName(path);
+
         public static readonly IBuildActions Instance = new SystemBuildActions();
     }
 }

csharp/autobuilder/Semmle.Autobuild/BuildActions.cs

@@ -1,7 +1,6 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
-using Semmle.Util;
 using Semmle.Util.Logging;
 
 namespace Semmle.Autobuild
@@ -32,7 +31,7 @@ public BuildScript Analyse(Autobuilder builder)
 
             var extensions = builder.Actions.IsWindows() ? winExtensions : linuxExtensions;
             var scripts = buildScripts.SelectMany(s => extensions.Select(e => s + e));
-            var scriptPath = builder.Paths.Where(p => scripts.Any(p.ToLower().EndsWith)).OrderBy(p => p.Length).FirstOrDefault();
+            var scriptPath = builder.Paths.Where(p => scripts.Any(p.Item1.ToLower().EndsWith)).OrderBy(p => p.Item2).Select(p => p.Item1).FirstOrDefault();
 
             if (scriptPath == null)
                 return BuildScript.Failure;
@@ -41,12 +40,12 @@ public BuildScript Analyse(Autobuilder builder)
             chmod.RunCommand("/bin/chmod", $"u+x {scriptPath}");
             var chmodScript = builder.Actions.IsWindows() ? BuildScript.Success : BuildScript.Try(chmod.Script);
 
-            var path = Path.GetDirectoryName(scriptPath);
+            var dir = builder.Actions.GetDirectoryName(scriptPath);
 
             // A specific .NET Core version may be required
             return chmodScript & DotNetRule.WithDotNet(builder, dotNet =>
             {
-                var command = new CommandBuilder(builder.Actions, path, dotNet?.Environment);
+                var command = new CommandBuilder(builder.Actions, dir, dotNet?.Environment);
 
                 // A specific Visual Studio version may be required
                 var vsTools = MsBuildRule.GetVcVarsBatFile(builder);

csharp/autobuilder/Semmle.Autobuild/BuildCommandAutoRule.cs

@@ -4,6 +4,7 @@
 using Newtonsoft.Json.Linq;
 using System.Collections.Generic;
 using System.IO;
+using Semmle.Util;
 
 namespace Semmle.Autobuild
 {
@@ -15,46 +16,36 @@ class DotNetRule : IBuildRule
     {
         public BuildScript Analyse(Autobuilder builder)
         {
-            builder.Log(Severity.Info, "Attempting to build using .NET Core");
-
-            var projects = builder.SolutionsToBuild.Any()
-                ? builder.SolutionsToBuild.SelectMany(s => s.Projects).ToArray()
-                : builder.GetExtensions(Language.CSharp.ProjectExtension).Select(p => new Project(builder, p)).ToArray();
+            if (!builder.ProjectsOrSolutionsToBuild.Any())
+                return BuildScript.Failure;
 
-            var notDotNetProject = projects.FirstOrDefault(p => !p.DotNetProject);
+            var notDotNetProject = builder.ProjectsOrSolutionsToBuild.
+                SelectMany(p => Enumerators.Singleton(p).Concat(p.IncludedProjects)).
+                OfType<Project>().
+                FirstOrDefault(p => !p.DotNetProject);
             if (notDotNetProject != null)
             {
                 builder.Log(Severity.Info, "Not using .NET Core because of incompatible project {0}", notDotNetProject);
                 return BuildScript.Failure;
             }
 
-            if (!builder.SolutionsToBuild.Any())
-                // Attempt dotnet build in root folder
-                return WithDotNet(builder, dotNet =>
-                    {
-                        var info = GetInfoCommand(builder.Actions, dotNet);
-                        var clean = GetCleanCommand(builder.Actions, dotNet).Script;
-                        var restore = GetRestoreCommand(builder.Actions, dotNet).Script;
-                        var build = GetBuildCommand(builder, dotNet).Script;
-                        return info & clean & BuildScript.Try(restore) & build;
-                    });
+            builder.Log(Severity.Info, "Attempting to build using .NET Core");
 
-            // Attempt dotnet build on each solution
             return WithDotNet(builder, dotNet =>
                 {
                     var ret = GetInfoCommand(builder.Actions, dotNet);
-                    foreach (var solution in builder.SolutionsToBuild)
+                    foreach (var projectOrSolution in builder.ProjectsOrSolutionsToBuild)
                     {
                         var cleanCommand = GetCleanCommand(builder.Actions, dotNet);
-                        cleanCommand.QuoteArgument(solution.Path);
+                        cleanCommand.QuoteArgument(projectOrSolution.FullPath);
                         var clean = cleanCommand.Script;
 
                         var restoreCommand = GetRestoreCommand(builder.Actions, dotNet);
-                        restoreCommand.QuoteArgument(solution.Path);
+                        restoreCommand.QuoteArgument(projectOrSolution.FullPath);
                         var restore = restoreCommand.Script;
 
                         var buildCommand = GetBuildCommand(builder, dotNet);
-                        buildCommand.QuoteArgument(solution.Path);
+                        buildCommand.QuoteArgument(projectOrSolution.FullPath);
                         var build = buildCommand.Script;
 
                         ret &= clean & BuildScript.Try(restore) & build;
@@ -110,7 +101,7 @@ static BuildScript DownloadDotNet(Autobuilder builder, string installDir)
             // See https://docs.microsoft.com/en-us/dotnet/core/tools/global-json
             var installScript = BuildScript.Success;
             var validGlobalJson = false;
-            foreach (var path in builder.Paths.Where(p => p.EndsWith("global.json", StringComparison.Ordinal)))
+            foreach (var path in builder.Paths.Select(p => p.Item1).Where(p => p.EndsWith("global.json", StringComparison.Ordinal)))
             {
                 string version;
                 try

csharp/autobuilder/Semmle.Autobuild/DotNetRule.cs

@@ -8,9 +8,6 @@ public sealed class Language
         public bool ProjectFileHasThisLanguage(string path) =>
             System.IO.Path.GetExtension(path) == ProjectExtension;
 
-        public static bool IsProjectFileForAnySupportedLanguage(string path) =>
-            Cpp.ProjectFileHasThisLanguage(path) || CSharp.ProjectFileHasThisLanguage(path);
-
         public readonly string ProjectExtension;
 
         private Language(string extension)