C#: Only set UseSharedCompilation=false in autobuilder when needed

Since we are now able to trace shared compilation builds on Linux and macOS
(starting from .NET Core 3), and always were able to on Windows, there is
no need to set `UseSharedCompilation=false` in those cases. This may have a
positive performance impact, as shared compilation is generally faster then
non-shared compilation.

Author: hvitved

csharp/autobuilder/Semmle.Autobuild.Tests/BuildScripts.cs

@@ -185,7 +185,7 @@ public class BuildScriptTests
         // Records the arguments passed to StartCallback.
         IList<string> StartCallbackIn = new List<string>();
 
-        void StartCallback(string s)
+        void StartCallback(string s, bool silent)
         {
             StartCallbackIn.Add(s);
         }
@@ -194,7 +194,7 @@ void StartCallback(string s)
         IList<string> EndCallbackIn = new List<string>();
         IList<int> EndCallbackReturn = new List<int>();
 
-        void EndCallback(int ret, string s)
+        void EndCallback(int ret, string s, bool silent)
         {
             EndCallbackReturn.Add(ret);
             EndCallbackIn.Add(s);
@@ -203,7 +203,7 @@ void EndCallback(int ret, string s)
         [Fact]
         public void TestBuildCommand()
         {
-            var cmd = BuildScript.Create("abc", "def ghi", null, null);
+            var cmd = BuildScript.Create("abc", "def ghi", false, null, null);
 
             Actions.RunProcess["abc def ghi"] = 1;
             cmd.Run(Actions, StartCallback, EndCallback);
@@ -216,7 +216,7 @@ public void TestBuildCommand()
         [Fact]
         public void TestAnd1()
         {
-            var cmd = BuildScript.Create("abc", "def ghi", null, null) & BuildScript.Create("odasa", null, null, null);
+            var cmd = BuildScript.Create("abc", "def ghi", false, null, null) & BuildScript.Create("odasa", null, false, null, null);
 
             Actions.RunProcess["abc def ghi"] = 1;
             cmd.Run(Actions, StartCallback, EndCallback);
@@ -230,7 +230,7 @@ public void TestAnd1()
         [Fact]
         public void TestAnd2()
         {
-            var cmd = BuildScript.Create("odasa", null, null, null) & BuildScript.Create("abc", "def ghi", null, null);
+            var cmd = BuildScript.Create("odasa", null, false, null, null) & BuildScript.Create("abc", "def ghi", false, null, null);
 
             Actions.RunProcess["abc def ghi"] = 1;
             Actions.RunProcess["odasa "] = 0;
@@ -250,7 +250,7 @@ public void TestAnd2()
         [Fact]
         public void TestOr1()
         {
-            var cmd = BuildScript.Create("odasa", null, null, null) | BuildScript.Create("abc", "def ghi", null, null);
+            var cmd = BuildScript.Create("odasa", null, false, null, null) | BuildScript.Create("abc", "def ghi", false, null, null);
 
             Actions.RunProcess["abc def ghi"] = 1;
             Actions.RunProcess["odasa "] = 0;
@@ -266,7 +266,7 @@ public void TestOr1()
         [Fact]
         public void TestOr2()
         {
-            var cmd = BuildScript.Create("abc", "def ghi", null, null) | BuildScript.Create("odasa", null, null, null);
+            var cmd = BuildScript.Create("abc", "def ghi", false, null, null) | BuildScript.Create("odasa", null, false, null, null);
 
             Actions.RunProcess["abc def ghi"] = 1;
             Actions.RunProcess["odasa "] = 0;
@@ -375,7 +375,7 @@ public void TestDefaultCSharpAutoBuilder()
             Actions.RunProcess["cmd.exe /C dotnet --info"] = 0;
             Actions.RunProcess["cmd.exe /C dotnet clean test.csproj"] = 0;
             Actions.RunProcess["cmd.exe /C dotnet restore test.csproj"] = 0;
-            Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --auto dotnet build --no-incremental /p:UseSharedCompilation=false test.csproj"] = 0;
+            Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --auto dotnet build --no-incremental test.csproj"] = 0;
             Actions.RunProcess[@"cmd.exe /C C:\codeql\tools\java\bin\java -jar C:\codeql\csharp\tools\extractor-asp.jar ."] = 0;
             Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --xml --extensions config csproj props xml"] = 0;
             Actions.FileExists["csharp.log"] = true;
@@ -404,6 +404,9 @@ public void TestLinuxCSharpAutoBuilder()
             Actions.RunProcess["dotnet --info"] = 0;
             Actions.RunProcess["dotnet clean test.csproj"] = 0;
             Actions.RunProcess["dotnet restore test.csproj"] = 0;
+            Actions.RunProcess["dotnet --list-runtimes"] = 0;
+            Actions.RunProcessOut["dotnet --list-runtimes"] = @"Microsoft.AspNetCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.AspNetCore.App]
+Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.App]";
             Actions.RunProcess[@"C:\odasa/tools/odasa index --auto dotnet build --no-incremental /p:UseSharedCompilation=false test.csproj"] = 0;
             Actions.RunProcess[@"C:\codeql\tools\java/bin/java -jar C:\codeql\csharp/tools/extractor-asp.jar ."] = 0;
             Actions.RunProcess[@"C:\odasa/tools/odasa index --xml --extensions config csproj props xml"] = 0;
@@ -424,7 +427,7 @@ public void TestLinuxCSharpAutoBuilder()
             Actions.LoadXml["test.csproj"] = xml;
 
             var autobuilder = CreateAutoBuilder("csharp", false);
-            TestAutobuilderScript(autobuilder, 0, 6);
+            TestAutobuilderScript(autobuilder, 0, 7);
         }
 
         [Fact]
@@ -748,7 +751,7 @@ public void TestWindowCSharpMsBuild()
             TestAutobuilderScript(autobuilder, 0, 6);
         }
 
-    [Fact]
+        [Fact]
         public void TestWindowCSharpMsBuildMultipleSolutions()
         {
             Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\csharp\nuget\nuget.exe restore test1.csproj"] = 0;
@@ -874,6 +877,9 @@ public void TestSkipNugetDotnet()
             Actions.RunProcess["dotnet --info"] = 0;
             Actions.RunProcess["dotnet clean test.csproj"] = 0;
             Actions.RunProcess["dotnet restore test.csproj"] = 0;
+            Actions.RunProcess["dotnet --list-runtimes"] = 0;
+            Actions.RunProcessOut["dotnet --list-runtimes"] = @"Microsoft.AspNetCore.App 2.1.3 [/usr/local/share/dotnet/shared/Microsoft.AspNetCore.App]
+Microsoft.NETCore.App 2.1.3 [/usr/local/share/dotnet/shared/Microsoft.NETCore.App]";
             Actions.RunProcess[@"C:\odasa/tools/odasa index --auto dotnet build --no-incremental /p:UseSharedCompilation=false --no-restore test.csproj"] = 0;
             Actions.RunProcess[@"C:\codeql\tools\java/bin/java -jar C:\codeql\csharp/tools/extractor-asp.jar ."] = 0;
             Actions.RunProcess[@"C:\odasa/tools/odasa index --xml --extensions config csproj props xml"] = 0;
@@ -894,7 +900,7 @@ public void TestSkipNugetDotnet()
             Actions.LoadXml["test.csproj"] = xml;
 
             var autobuilder = CreateAutoBuilder("csharp", false, dotnetArguments: "--no-restore");  // nugetRestore=false does not work for now.
-            TestAutobuilderScript(autobuilder, 0, 6);
+            TestAutobuilderScript(autobuilder, 0, 7);
         }
 
         [Fact]
@@ -909,7 +915,10 @@ public void TestDotnetVersionNotInstalled()
             Actions.RunProcess[@"C:\Project/.dotnet/dotnet --info"] = 0;
             Actions.RunProcess[@"C:\Project/.dotnet/dotnet clean test.csproj"] = 0;
             Actions.RunProcess[@"C:\Project/.dotnet/dotnet restore test.csproj"] = 0;
-            Actions.RunProcess[@"C:\odasa/tools/odasa index --auto C:\Project/.dotnet/dotnet build --no-incremental /p:UseSharedCompilation=false test.csproj"] = 0;
+            Actions.RunProcess[@"C:\Project/.dotnet/dotnet --list-runtimes"] = 0;
+            Actions.RunProcessOut[@"C:\Project/.dotnet/dotnet --list-runtimes"] = @"Microsoft.AspNetCore.App 3.0.0 [/usr/local/share/dotnet/shared/Microsoft.AspNetCore.App]
+Microsoft.NETCore.App 3.0.0 [/usr/local/share/dotnet/shared/Microsoft.NETCore.App]";
+            Actions.RunProcess[@"C:\odasa/tools/odasa index --auto C:\Project/.dotnet/dotnet build --no-incremental test.csproj"] = 0;
             Actions.RunProcess[@"C:\codeql\tools\java/bin/java -jar C:\codeql\csharp/tools/extractor-asp.jar ."] = 0;
             Actions.RunProcess[@"C:\odasa/tools/odasa index --xml --extensions config csproj props xml"] = 0;
             Actions.FileExists["csharp.log"] = true;
@@ -930,21 +939,27 @@ public void TestDotnetVersionNotInstalled()
             Actions.LoadXml["test.csproj"] = xml;
 
             var autobuilder = CreateAutoBuilder("csharp", false, dotnetVersion: "2.1.3");
-            TestAutobuilderScript(autobuilder, 0, 11);
+            TestAutobuilderScript(autobuilder, 0, 12);
         }
 
         [Fact]
         public void TestDotnetVersionAlreadyInstalled()
         {
             Actions.RunProcess["dotnet --list-sdks"] = 0;
-            Actions.RunProcessOut["dotnet --list-sdks"] = "2.1.3 [C:\\Program Files\\dotnet\\sdks]\n2.1.4 [C:\\Program Files\\dotnet\\sdks]";
+            Actions.RunProcessOut["dotnet --list-sdks"] = @"2.1.3 [C:\Program Files\dotnet\sdks]
+2.1.4 [C:\Program Files\dotnet\sdks]";
             Actions.RunProcess[@"curl -L -sO https://dot.net/v1/dotnet-install.sh"] = 0;
             Actions.RunProcess[@"chmod u+x dotnet-install.sh"] = 0;
             Actions.RunProcess[@"./dotnet-install.sh --channel release --version 2.1.3 --install-dir C:\Project/.dotnet"] = 0;
             Actions.RunProcess[@"rm dotnet-install.sh"] = 0;
             Actions.RunProcess[@"C:\Project/.dotnet/dotnet --info"] = 0;
             Actions.RunProcess[@"C:\Project/.dotnet/dotnet clean test.csproj"] = 0;
             Actions.RunProcess[@"C:\Project/.dotnet/dotnet restore test.csproj"] = 0;
+            Actions.RunProcess[@"C:\Project/.dotnet/dotnet --list-runtimes"] = 0;
+            Actions.RunProcessOut[@"C:\Project/.dotnet/dotnet --list-runtimes"] = @"Microsoft.AspNetCore.App 2.1.3 [/usr/local/share/dotnet/shared/Microsoft.AspNetCore.App]
+Microsoft.AspNetCore.App 2.1.4 [/usr/local/share/dotnet/shared/Microsoft.AspNetCore.App]
+Microsoft.NETCore.App 2.1.3 [/usr/local/share/dotnet/shared/Microsoft.NETCore.App]
+Microsoft.NETCore.App 2.1.4 [/usr/local/share/dotnet/shared/Microsoft.NETCore.App]";
             Actions.RunProcess[@"C:\odasa/tools/odasa index --auto C:\Project/.dotnet/dotnet build --no-incremental /p:UseSharedCompilation=false test.csproj"] = 0;
             Actions.RunProcess[@"C:\codeql\tools\java/bin/java -jar C:\codeql\csharp/tools/extractor-asp.jar ."] = 0;
             Actions.RunProcess[@"C:\odasa/tools/odasa index --xml --extensions config csproj props xml"] = 0;
@@ -966,7 +981,7 @@ public void TestDotnetVersionAlreadyInstalled()
             Actions.LoadXml["test.csproj"] = xml;
 
             var autobuilder = CreateAutoBuilder("csharp", false, dotnetVersion: "2.1.3");
-            TestAutobuilderScript(autobuilder, 0, 11);
+            TestAutobuilderScript(autobuilder, 0, 12);
         }
 
         [Fact]
@@ -979,7 +994,7 @@ public void TestDotnetVersionWindows()
             Actions.RunProcess[@"cmd.exe /C C:\Project\.dotnet\dotnet --info"] = 0;
             Actions.RunProcess[@"cmd.exe /C C:\Project\.dotnet\dotnet clean test.csproj"] = 0;
             Actions.RunProcess[@"cmd.exe /C C:\Project\.dotnet\dotnet restore test.csproj"] = 0;
-            Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --auto C:\Project\.dotnet\dotnet build --no-incremental /p:UseSharedCompilation=false test.csproj"] = 0;
+            Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --auto C:\Project\.dotnet\dotnet build --no-incremental test.csproj"] = 0;
             Actions.RunProcess[@"cmd.exe /C C:\codeql\tools\java\bin\java -jar C:\codeql\csharp\tools\extractor-asp.jar ."] = 0;
             Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --xml --extensions config csproj props xml"] = 0;
             Actions.FileExists["csharp.log"] = true;

csharp/autobuilder/Semmle.Autobuild/Autobuilder.cs

@@ -214,8 +214,17 @@ public int AttemptBuild()
             if (Options.IgnoreErrors)
                 script |= BuildScript.Success;
 
-            void startCallback(string s) => Log(Severity.Info, $"\nRunning {s}");
-            void exitCallback(int ret, string msg) => Log(Severity.Info, $"Exit code {ret}{(string.IsNullOrEmpty(msg) ? "" : $": {msg}")}");
+            void startCallback(string s, bool silent)
+            {
+                if (!silent) Log(Severity.Info, $"\nRunning {s}");
+            }
+
+            void exitCallback(int ret, string msg, bool silent)
+            {
+                if (!silent)
+                    Log(Severity.Info, $"Exit code {ret}{(string.IsNullOrEmpty(msg) ? "" : $": {msg}")}");
+            }
+
             return script.Run(Actions, startCallback, exitCallback);
         }
 

csharp/autobuilder/Semmle.Autobuild/BuildActions.cs

@@ -141,12 +141,8 @@ ProcessStartInfo GetProcessStartInfo(string exe, string arguments, string workin
                 pi.WorkingDirectory = workingDirectory;
 
             // Environment variables can only be used when not redirecting stdout
-            if (!redirectStandardOutput)
-            {
-                pi.Environment["UseSharedCompilation"] = "false";
-                if (environment != null)
-                    environment.ForEach(kvp => pi.Environment[kvp.Key] = kvp.Value);
-            }
+            if (!redirectStandardOutput && environment != null)
+                environment.ForEach(kvp => pi.Environment[kvp.Key] = kvp.Value);
             return pi;
         }
 

csharp/autobuilder/Semmle.Autobuild/BuildScript.cs

@@ -25,7 +25,7 @@ public abstract class BuildScript
         /// an exit message.
         /// </param>
         /// <returns>The exit code from this build script.</returns>
-        public abstract int Run(IBuildActions actions, Action<string> startCallback, Action<int, string> exitCallBack);
+        public abstract int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack);
 
         /// <summary>
         /// Run this build command.
@@ -44,33 +44,36 @@ public abstract class BuildScript
         /// </param>
         /// <param name="stdout">Contents of standard out.</param>
         /// <returns>The exit code from this build script.</returns>
-        public abstract int Run(IBuildActions actions, Action<string> startCallback, Action<int, string> exitCallBack, out IList<string> stdout);
+        public abstract int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, out IList<string> stdout);
 
         class BuildCommand : BuildScript
         {
             readonly string exe, arguments, workingDirectory;
             readonly IDictionary<string, string> environment;
+            readonly bool silent;
 
             /// <summary>
             /// Create a simple build command.
             /// </summary>
             /// <param name="exe">The executable to run.</param>
             /// <param name="argumentsOpt">The arguments to the executable, or null.</param>
+            /// <param name="silent">Whether this command should run silently.</param>
             /// <param name="workingDirectory">The working directory (<code>null</code> for current directory).</param>
             /// <param name="environment">Additional environment variables.</param>
-            public BuildCommand(string exe, string argumentsOpt, string workingDirectory = null, IDictionary<string, string> environment = null)
+            public BuildCommand(string exe, string argumentsOpt, bool silent, string workingDirectory = null, IDictionary<string, string> environment = null)
             {
                 this.exe = exe;
                 this.arguments = argumentsOpt ?? "";
+                this.silent = silent;
                 this.workingDirectory = workingDirectory;
                 this.environment = environment;
             }
 
             public override string ToString() => exe + " " + arguments;
 
-            public override int Run(IBuildActions actions, Action<string> startCallback, Action<int, string> exitCallBack)
+            public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack)
             {
-                startCallback(this.ToString());
+                startCallback(this.ToString(), silent);
                 var ret = 1;
                 var retMessage = "";
                 try
@@ -83,13 +86,13 @@ public override int Run(IBuildActions actions, Action<string> startCallback, Act
                     retMessage = ex.Message;
                 }
 
-                exitCallBack(ret, retMessage);
+                exitCallBack(ret, retMessage, silent);
                 return ret;
             }
 
-            public override int Run(IBuildActions actions, Action<string> startCallback, Action<int, string> exitCallBack, out IList<string> stdout)
+            public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, out IList<string> stdout)
             {
-                startCallback(this.ToString());
+                startCallback(this.ToString(), silent);
                 var ret = 1;
                 var retMessage = "";
                 try
@@ -102,7 +105,7 @@ public override int Run(IBuildActions actions, Action<string> startCallback, Act
                     retMessage = ex.Message;
                     stdout = new string[0];
                 }
-                exitCallBack(ret, retMessage);
+                exitCallBack(ret, retMessage, silent);
                 return ret;
             }
 
@@ -116,9 +119,9 @@ public ReturnBuildCommand(Func<IBuildActions, int> func)
                 this.func = func;
             }
 
-            public override int Run(IBuildActions actions, Action<string> startCallback, Action<int, string> exitCallBack) => func(actions);
+            public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack) => func(actions);
 
-            public override int Run(IBuildActions actions, Action<string> startCallback, Action<int, string> exitCallBack, out IList<string> stdout)
+            public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, out IList<string> stdout)
             {
                 stdout = new string[0];
                 return func(actions);
@@ -142,7 +145,7 @@ public BindBuildScript(BuildScript s1, Func<int, BuildScript> s2)
                 this.s2b = s2;
             }
 
-            public override int Run(IBuildActions actions, Action<string> startCallback, Action<int, string> exitCallBack)
+            public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack)
             {
                 int ret1;
                 if (s2a != null)
@@ -155,7 +158,7 @@ public override int Run(IBuildActions actions, Action<string> startCallback, Act
                 return s2b(ret1).Run(actions, startCallback, exitCallBack);
             }
 
-            public override int Run(IBuildActions actions, Action<string> startCallback, Action<int, string> exitCallBack, out IList<string> stdout)
+            public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, out IList<string> stdout)
             {
                 var ret1 = s1.Run(actions, startCallback, exitCallBack, out var stdout1);
                 var ret2 = (s2a != null ? s2a(stdout1, ret1) : s2b(ret1)).Run(actions, startCallback, exitCallBack, out var stdout2);
@@ -171,10 +174,11 @@ public override int Run(IBuildActions actions, Action<string> startCallback, Act
         /// Creates a simple build script that runs the specified exe.
         /// </summary>
         /// <param name="argumentsOpt">The arguments to the executable, or null.</param>
+        /// <param name="silent">Whether the executable should run silently.</param>
         /// <param name="workingDirectory">The working directory (<code>null</code> for current directory).</param>
         /// <param name="environment">Additional environment variables.</param>
-        public static BuildScript Create(string exe, string argumentsOpt, string workingDirectory, IDictionary<string, string> environment) =>
-            new BuildCommand(exe, argumentsOpt, workingDirectory, environment);
+        public static BuildScript Create(string exe, string argumentsOpt, bool silent, string workingDirectory, IDictionary<string, string> environment) =>
+            new BuildCommand(exe, argumentsOpt, silent, workingDirectory, environment);
 
         /// <summary>
         /// Creates a simple build script that runs the specified function.