Force the nuget download of dotnet framework reference assemblies

Author: tamasvajk

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -47,9 +47,12 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
             this.progressMonitor = new ProgressMonitor(logger);
             this.sourceDir = new DirectoryInfo(srcDir);
 
+            packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName));
+            tempWorkingDirectory = new TemporaryDirectory(FileUtils.GetTemporaryWorkingDirectory(out cleanupTempWorkingDirectory));
+
             try
             {
-                this.dotnet = DotNet.Make(options, progressMonitor);
+                this.dotnet = DotNet.Make(options, progressMonitor, tempWorkingDirectory);
             }
             catch
             {
@@ -59,8 +62,6 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
 
             this.progressMonitor.FindingFiles(srcDir);
 
-            packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName));
-            tempWorkingDirectory = new TemporaryDirectory(FileUtils.GetTemporaryWorkingDirectory(out cleanupTempWorkingDirectory));
 
             var allFiles = GetAllFiles();
             var binaryFileExtensions = new HashSet<string>(new[] { ".dll", ".exe" }); // TODO: add more binary file extensions.
@@ -537,11 +538,11 @@ private void AnalyseProject(FileInfo project)
 
         }
 
-        private bool RestoreProject(string project, string? pathToNugetConfig = null) =>
-            dotnet.RestoreProjectToDirectory(project, packageDirectory.DirInfo.FullName, pathToNugetConfig);
+        private bool RestoreProject(string project, bool forceDotnetRefAssemblyFetching, string? pathToNugetConfig = null) =>
+            dotnet.RestoreProjectToDirectory(project, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching, pathToNugetConfig);
 
         private bool RestoreSolution(string solution, out IEnumerable<string> projects) =>
-            dotnet.RestoreSolutionToDirectory(solution, packageDirectory.DirInfo.FullName, out projects);
+            dotnet.RestoreSolutionToDirectory(solution, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: true, out projects);
 
         /// <summary>
         /// Executes `dotnet restore` on all solution files in solutions.
@@ -567,7 +568,7 @@ private void RestoreProjects(IEnumerable<string> projects)
         {
             Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, project =>
             {
-                RestoreProject(project);
+                RestoreProject(project, forceDotnetRefAssemblyFetching: true);
             });
         }
 
@@ -612,7 +613,7 @@ private void DownloadMissingPackages(List<FileInfo> allFiles)
                     return;
                 }
 
-                success = RestoreProject(tempDir.DirInfo.FullName, nugetConfig);
+                success = RestoreProject(tempDir.DirInfo.FullName, forceDotnetRefAssemblyFetching: false, pathToNugetConfig: nugetConfig);
                 // TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
                 if (!success)
                 {

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs

@@ -3,6 +3,7 @@
 using System.IO;
 using System.Linq;
 using System.Text.RegularExpressions;
+using Semmle.Util;
 
 namespace Semmle.Extraction.CSharp.DependencyFetching
 {
@@ -13,19 +14,21 @@ internal partial class DotNet : IDotNet
     {
         private readonly IDotNetCliInvoker dotnetCliInvoker;
         private readonly ProgressMonitor progressMonitor;
+        private readonly TemporaryDirectory? tempWorkingDirectory;
 
-        private DotNet(IDotNetCliInvoker dotnetCliInvoker, ProgressMonitor progressMonitor)
+        private DotNet(IDotNetCliInvoker dotnetCliInvoker, ProgressMonitor progressMonitor, TemporaryDirectory? tempWorkingDirectory = null)
         {
             this.progressMonitor = progressMonitor;
+            this.tempWorkingDirectory = tempWorkingDirectory;
             this.dotnetCliInvoker = dotnetCliInvoker;
             Info();
         }
 
-        private DotNet(IDependencyOptions options, ProgressMonitor progressMonitor) : this(new DotNetCliInvoker(progressMonitor, Path.Combine(options.DotNetPath ?? string.Empty, "dotnet")), progressMonitor) { }
+        private DotNet(IDependencyOptions options, ProgressMonitor progressMonitor, TemporaryDirectory tempWorkingDirectory) : this(new DotNetCliInvoker(progressMonitor, Path.Combine(options.DotNetPath ?? string.Empty, "dotnet")), progressMonitor, tempWorkingDirectory) { }
 
         internal static IDotNet Make(IDotNetCliInvoker dotnetCliInvoker, ProgressMonitor progressMonitor) => new DotNet(dotnetCliInvoker, progressMonitor);
 
-        public static IDotNet Make(IDependencyOptions options, ProgressMonitor progressMonitor) => new DotNet(options, progressMonitor);
+        public static IDotNet Make(IDependencyOptions options, ProgressMonitor progressMonitor, TemporaryDirectory tempWorkingDirectory) => new DotNet(options, progressMonitor, tempWorkingDirectory);
 
         private void Info()
         {
@@ -37,12 +40,29 @@ private void Info()
             }
         }
 
-        private static string GetRestoreArgs(string projectOrSolutionFile, string packageDirectory) =>
-            $"restore --no-dependencies \"{projectOrSolutionFile}\" --packages \"{packageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true";
+        private string GetRestoreArgs(string projectOrSolutionFile, string packageDirectory, bool forceDotnetRefAssemblyFetching)
+        {
+            var args = $"restore --no-dependencies \"{projectOrSolutionFile}\" --packages \"{packageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true";
+
+            if (forceDotnetRefAssemblyFetching)
+            {
+                // Ugly hack: we set the TargetFrameworkRootPath and NetCoreTargetingPackRoot properties to an empty folder:
+                var path = ".empty";
+                if (tempWorkingDirectory != null)
+                {
+                    path = Path.Combine(tempWorkingDirectory.ToString(), "emptyFakeDotnetRoot");
+                    Directory.CreateDirectory(path);
+                }
+
+                args += $" /p:TargetFrameworkRootPath=\"{path}\" /p:NetCoreTargetingPackRoot=\"{path}\"";
+            }
+
+            return args;
+        }
 
-        public bool RestoreProjectToDirectory(string projectFile, string packageDirectory, string? pathToNugetConfig = null)
+        public bool RestoreProjectToDirectory(string projectFile, string packageDirectory, bool forceDotnetRefAssemblyFetching, string? pathToNugetConfig = null)
         {
-            var args = GetRestoreArgs(projectFile, packageDirectory);
+            var args = GetRestoreArgs(projectFile, packageDirectory, forceDotnetRefAssemblyFetching);
             if (pathToNugetConfig != null)
             {
                 args += $" --configfile \"{pathToNugetConfig}\"";
@@ -51,9 +71,9 @@ public bool RestoreProjectToDirectory(string projectFile, string packageDirector
             return dotnetCliInvoker.RunCommand(args);
         }
 
-        public bool RestoreSolutionToDirectory(string solutionFile, string packageDirectory, out IEnumerable<string> projects)
+        public bool RestoreSolutionToDirectory(string solutionFile, string packageDirectory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> projects)
         {
-            var args = GetRestoreArgs(solutionFile, packageDirectory);
+            var args = GetRestoreArgs(solutionFile, packageDirectory, forceDotnetRefAssemblyFetching);
             args += " --verbosity normal";
             if (dotnetCliInvoker.RunCommand(args, out var output))
             {

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs

@@ -4,8 +4,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 {
     internal interface IDotNet
     {
-        bool RestoreProjectToDirectory(string project, string directory, string? pathToNugetConfig = null);
-        bool RestoreSolutionToDirectory(string solutionFile, string packageDirectory, out IEnumerable<string> projects);
+        bool RestoreProjectToDirectory(string project, string directory, bool forceDotnetRefAssemblyFetching, string? pathToNugetConfig = null);
+        bool RestoreSolutionToDirectory(string solutionFile, string packageDirectory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> projects);
         bool New(string folder);
         bool AddPackage(string folder, string package);
         IList<string> GetListedRuntimes();

csharp/extractor/Semmle.Extraction.Tests/DotNet.cs

@@ -99,7 +99,7 @@ public void TestDotnetRestoreProjectToDirectory1()
             var dotnet = MakeDotnet(dotnetCliInvoker);
 
             // Execute
-            dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages");
+            dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", false);
 
             // Verify
             var lastArgs = dotnetCliInvoker.GetLastArgs();
@@ -114,7 +114,7 @@ public void TestDotnetRestoreProjectToDirectory2()
             var dotnet = MakeDotnet(dotnetCliInvoker);
 
             // Execute
-            dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", "myconfig.config");
+            dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", false, "myconfig.config");
 
             // Verify
             var lastArgs = dotnetCliInvoker.GetLastArgs();
@@ -129,7 +129,7 @@ public void TestDotnetRestoreSolutionToDirectory1()
             var dotnet = MakeDotnet(dotnetCliInvoker);
 
             // Execute
-            dotnet.RestoreSolutionToDirectory("mysolution.sln", "mypackages", out var projects);
+            dotnet.RestoreSolutionToDirectory("mysolution.sln", "mypackages", false, out var projects);
 
             // Verify
             var lastArgs = dotnetCliInvoker.GetLastArgs();
@@ -148,7 +148,7 @@ public void TestDotnetRestoreSolutionToDirectory2()
             dotnetCliInvoker.Success = false;
 
             // Execute
-            dotnet.RestoreSolutionToDirectory("mysolution.sln", "mypackages", out var projects);
+            dotnet.RestoreSolutionToDirectory("mysolution.sln", "mypackages", false, out var projects);
 
             // Verify
             var lastArgs = dotnetCliInvoker.GetLastArgs();

csharp/extractor/Semmle.Extraction.Tests/Runtime.cs

@@ -19,9 +19,9 @@ public DotNetStub(IList<string> runtimes, IList<string> sdks)
 
         public bool New(string folder) => true;
 
-        public bool RestoreProjectToDirectory(string project, string directory, string? pathToNugetConfig = null) => true;
+        public bool RestoreProjectToDirectory(string project, string directory, bool forceDotnetRefAssemblyFetching, string? pathToNugetConfig = null) => true;
 
-        public bool RestoreSolutionToDirectory(string solution, string directory, out IEnumerable<string> projects)
+        public bool RestoreSolutionToDirectory(string solution, string directory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> projects)
         {
             projects = Array.Empty<string>();
             return true;