Merge pull request #327 from hvitved/csharp/compiler-flag-linux

calumgrant · web-flow · commit 6811d527e17f · 2018-10-19T11:18:46.000+01:00
C#: Pass `--compiler` flag to extractor on Linux
diff --git a/csharp/config/tracer/linux/csharp-compiler-settings b/csharp/config/tracer/linux/csharp-compiler-settings
@@ -1,6 +1,8 @@
 **/mcs.exe:
 **/csc.exe:
   invoke ${env.SEMMLE_PLATFORM_TOOLS}/csharp/Semmle.Extraction.CSharp.Driver
+  prepend --compiler
+  prepend "${compiler}"
   prepend --cil
 **/bin/mono*:
 **/dotnet:
diff --git a/csharp/config/tracer/linux/extract-csharp.sh b/csharp/config/tracer/linux/extract-csharp.sh
@@ -9,7 +9,7 @@ do
   if [[ `basename -- "$i"` =~ csc.exe|mcs.exe|csc.dll ]]
   then
     echo extract-csharp.sh: exec $extractor --cil $@
-    exec "$extractor" --cil $@
+    exec "$extractor" --compiler $i --cil $@
   fi
 done
 
diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Analyser.cs b/csharp/extractor/Semmle.Extraction.CSharp/Analyser.cs
@@ -46,10 +46,12 @@ public Analyser(IProgressMonitor pm, ILogger logger)
         /// <param name="commandLineArguments">Arguments passed to csc.</param>
         /// <param name="compilationIn">The Roslyn compilation.</param>
         /// <param name="options">Extractor options.</param>
+        /// <param name="roslynArgs">The arguments passed to Roslyn.</param>
         public void Initialize(
             CSharpCommandLineArguments commandLineArguments,
             CSharpCompilation compilationIn,
-            Options options)
+            Options options,
+            string[] roslynArgs)
         {
             compilation = compilationIn;
 
@@ -58,7 +60,7 @@ public void Initialize(
 
             extractor = new Extraction.Extractor(false, GetOutputName(compilation, commandLineArguments), Logger);
 
-            LogDiagnostics();
+            LogDiagnostics(roslynArgs);
             SetReferencePaths();
 
             CompilationErrors += FilteredDiagnostics.Count();
@@ -113,7 +115,7 @@ public void InitializeStandalone(CSharpCompilation compilationIn, CommonOptions
             layout = new Layout();
             extractor = new Extraction.Extractor(true, null, Logger);
             this.options = options;
-            LogDiagnostics();
+            LogDiagnostics(null);
             SetReferencePaths();
         }
 
@@ -409,7 +411,8 @@ void AppendQuoted(StringBuilder sb, string s)
         /// Logs detailed information about this invocation,
         /// in the event that errors were detected.
         /// </summary>
-        public void LogDiagnostics()
+        /// <param name="roslynArgs">The arguments passed to Roslyn.</param>
+        public void LogDiagnostics(string[] roslynArgs)
         {
             Logger.Log(Severity.Info, "  Current working directory: {0}", Directory.GetCurrentDirectory());
             Logger.Log(Severity.Info, "  Extractor: {0}", Environment.GetCommandLineArgs().First());
@@ -438,6 +441,9 @@ public void LogDiagnostics()
             }
             Logger.Log(Severity.Info, sb.ToString());
 
+            if (roslynArgs != null)
+                Logger.Log(Severity.Info, $"  Arguments to Roslyn: {string.Join(' ', roslynArgs)}");
+
             foreach (var error in FilteredDiagnostics)
             {
                 Logger.Log(Severity.Error, "  Compilation error: {0}", error);
diff --git a/csharp/extractor/Semmle.Extraction.CSharp/CompilerVersion.cs b/csharp/extractor/Semmle.Extraction.CSharp/CompilerVersion.cs
@@ -1,5 +1,7 @@
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
+using System.Linq;
 using System.Runtime.InteropServices;
 
 namespace Semmle.Extraction.CSharp
@@ -53,24 +55,34 @@ public CompilerVersion(Options options)
                 var versionInfo = FileVersionInfo.GetVersionInfo(SpecifiedCompiler);
 
                 var compilerDir = Path.GetDirectoryName(SpecifiedCompiler);
-                bool known_compiler_name = versionInfo.OriginalFilename == "csc.exe" || versionInfo.OriginalFilename == "csc2.exe";
-                bool copyright_microsoft = versionInfo.LegalCopyright != null && versionInfo.LegalCopyright.Contains("Microsoft");
-                bool mscorlib_exists = File.Exists(Path.Combine(compilerDir, "mscorlib.dll"));
+                var knownCompilerNames = new Dictionary<string, string>
+                {
+                    { "csc.exe", "Microsoft" },
+                    { "csc2.exe", "Microsoft" },
+                    { "csc.dll", "Microsoft" },
+                    { "mcs.exe", "Novell" }
+                };
+                var mscorlibExists = File.Exists(Path.Combine(compilerDir, "mscorlib.dll"));
 
-                if (specifiedFramework == null && mscorlib_exists)
+                if (specifiedFramework == null && mscorlibExists)
                 {
                     specifiedFramework = compilerDir;
                 }
 
-                if (!known_compiler_name)
+                if (!knownCompilerNames.TryGetValue(versionInfo.OriginalFilename, out var vendor))
                 {
-                    SkipExtractionBecause("the exe name is not recognised");
+                    SkipExtractionBecause("the compiler name is not recognised");
+                    return;
                 }
-                else if (!copyright_microsoft)
+
+                if (versionInfo.LegalCopyright == null || !versionInfo.LegalCopyright.Contains(vendor))
                 {
-                    SkipExtractionBecause("the exe isn't copyright Microsoft");
+                    SkipExtractionBecause($"the compiler isn't copyright {vendor}, but instead {versionInfo.LegalCopyright ?? "<null>"}");
+                    return;
                 }
             }
+
+            ArgsWithResponse = AddDefaultResponse(CscRsp, options.CompilerArguments).ToArray();
         }
 
         void SkipExtractionBecause(string reason)
@@ -87,7 +99,7 @@ void SkipExtractionBecause(string reason)
         /// <summary>
         /// The file csc.rsp.
         /// </summary>
-        public string CscRsp => Path.Combine(FrameworkPath, csc_rsp);
+        string CscRsp => Path.Combine(FrameworkPath, csc_rsp);
 
         /// <summary>
         /// Should we skip extraction?
@@ -103,5 +115,25 @@ public bool SkipExtraction
         /// Gets additional reference directories - the compiler directory.
         /// </summary>
         public string AdditionalReferenceDirectories => SpecifiedCompiler != null ? Path.GetDirectoryName(SpecifiedCompiler) : null;
+
+        /// <summary>
+        /// Adds @csc.rsp to the argument list to mimic csc.exe.
+        /// </summary>
+        /// <param name="responseFile">The full pathname of csc.rsp.</param>
+        /// <param name="args">The other command line arguments.</param>
+        /// <returns>Modified list of arguments.</returns>
+        static IEnumerable<string> AddDefaultResponse(string responseFile, IEnumerable<string> args)
+        {
+            return SuppressDefaultResponseFile(args) && File.Exists(responseFile) ?
+                args :
+                new[] { "@" + responseFile }.Concat(args);
+        }
+
+        static bool SuppressDefaultResponseFile(IEnumerable<string> args)
+        {
+            return args.Any(arg => new[] { "/noconfig", "-noconfig" }.Contains(arg.ToLowerInvariant()));
+        }
+
+        public readonly string[] ArgsWithResponse;
     }
 }
diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Extractor.cs b/csharp/extractor/Semmle.Extraction.CSharp/Extractor.cs
@@ -87,11 +87,9 @@ public static ExitCode Run(string[] args)
                         return ExitCode.Ok;
                     }
 
-                    var argsWithResponse = AddDefaultResponse(compilerVersion.CscRsp, commandLineArguments.CompilerArguments);
-
                     var cwd = Directory.GetCurrentDirectory();
                     var compilerArguments = CSharpCommandLineParser.Default.Parse(
-                        argsWithResponse,
+                        compilerVersion.ArgsWithResponse,
                         cwd,
                         compilerVersion.FrameworkPath,
                         compilerVersion.AdditionalReferenceDirectories
@@ -128,7 +126,7 @@ public static ExitCode Run(string[] args)
                     {
                         logger.Log(Severity.Error, "  No source files");
                         ++analyser.CompilationErrors;
-                        analyser.LogDiagnostics();
+                        analyser.LogDiagnostics(compilerVersion.ArgsWithResponse);
                         return ExitCode.Failed;
                     }
 
@@ -146,7 +144,7 @@ public static ExitCode Run(string[] args)
                         // already.
                         );
 
-                    analyser.Initialize(compilerArguments, compilation, commandLineArguments);
+                    analyser.Initialize(compilerArguments, compilation, commandLineArguments, compilerVersion.ArgsWithResponse);
                     analyser.AnalyseReferences();
 
                     foreach (var tree in compilation.SyntaxTrees)
@@ -172,24 +170,6 @@ public static ExitCode Run(string[] args)
             }
         }
 
-        internal static bool SuppressDefaultResponseFile(IEnumerable<string> args)
-        {
-            return args.Any(arg => new[] { "/noconfig", "-noconfig" }.Contains(arg.ToLowerInvariant()));
-        }
-
-        /// <summary>
-        /// Adds @csc.rsp to the argument list to mimic csc.exe.
-        /// </summary>
-        /// <param name="responseFile">The full pathname of csc.rsp.</param>
-        /// <param name="args">The other command line arguments.</param>
-        /// <returns>Modified list of arguments.</returns>
-        static IEnumerable<string> AddDefaultResponse(string responseFile, IEnumerable<string> args)
-        {
-            return SuppressDefaultResponseFile(args) && File.Exists(responseFile) ?
-                args :
-                new[] { "@" + responseFile }.Concat(args);
-        }
-
         /// <summary>
         /// Gets the complete list of locations to locate references.
         /// </summary>
diff --git a/csharp/extractor/Semmle.Extraction/Context.cs b/csharp/extractor/Semmle.Extraction/Context.cs
@@ -171,6 +171,10 @@ public void PopulateAll()
                 {
                     populateQueue.Dequeue()();
                 }
+                catch (InternalError e)
+                {
+                    Extractor.Message(e.ExtractionMessage);
+                }
                 catch (Exception e)
                 {
                     Extractor.Message(new Message { severity = Severity.Error, exception = e, message = "Uncaught exception" });
diff --git a/csharp/extractor/Semmle.Extraction/TrapWriter.cs b/csharp/extractor/Semmle.Extraction/TrapWriter.cs
@@ -1,6 +1,7 @@
 using System;
 using System.IO;
 using System.IO.Compression;
+using System.Security.Cryptography;
 using System.Text;
 using Semmle.Util;
 using Semmle.Util.Logging;
@@ -170,21 +171,23 @@ public void Dispose()
                     WriterLazy.Value.Close();
                     if (TryMove(tmpFile, TrapFile))
                         return;
-                    else if (discardDuplicates)
+
+                    if (discardDuplicates)
                     {
                         FileUtils.TryDelete(tmpFile);
                         return;
                     }
 
-                    string root = TrapFile.Substring(0, TrapFile.Length - 8);   // Remove trailing ".trap.gz"
-
-                    // Loop until we find an available trap filename.
-                    for (int n = 0; n < 100; ++n)
+                    var existingHash = ComputeHash(TrapFile);
+                    var hash = ComputeHash(tmpFile);
+                    if (existingHash != hash)
                     {
-                        if (TryMove(tmpFile, string.Format("{0}.{1}.trap.gz", root, n)))
+                        var root = TrapFile.Substring(0, TrapFile.Length - 8); // Remove trailing ".trap.gz"
+                        if (TryMove(tmpFile, $"{root}-{hash}.trap.gz"))
                             return;
                     }
-                    Logger.Log(Severity.Error, "Failed to move the trap file from {0} to {1}", tmpFile, TrapFile);
+                    Logger.Log(Severity.Info, "Identical trap file for {0} already exists", TrapFile);
+                    FileUtils.TryDelete(tmpFile);
                 }
             }
             catch (Exception ex)
@@ -203,6 +206,22 @@ public void Emit(ITrapEmitter emitter)
         //#################### PRIVATE METHODS ####################
         #region
 
+        /// <summary>
+        /// Computes the hash of <paramref name="filePath"/>.
+        /// </summary>
+        static string ComputeHash(string filePath)
+        {
+            using (var fileStream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read))
+            using (var shaAlg = new SHA256Managed())
+            {
+                var sha = shaAlg.ComputeHash(fileStream);
+                var hex = new StringBuilder(sha.Length * 2);
+                foreach (var b in sha)
+                    hex.AppendFormat("{0:x2}", b);
+                return hex.ToString();
+            }
+        }
+
         class TrapBuilder : ITrapBuilder
         {
             readonly StreamWriter StreamWriter;

Original file line number	Diff line number	Diff line change
`@@ -171,6 +171,10 @@ public void PopulateAll()`
`171`	`171`	`{`
`172`	`172`	`populateQueue.Dequeue()();`
`173`	`173`	`}`
	`174`	`+ catch (InternalError e)`
	`175`	`+ {`
	`176`	`+ Extractor.Message(e.ExtractionMessage);`
	`177`	`+ }`
`174`	`178`	`catch (Exception e)`
`175`	`179`	`{`
`176`	`180`	`Extractor.Message(new Message { severity = Severity.Error, exception = e, message = "Uncaught exception" });`