Skip to content

Commit 8f0f696

Browse files
tamasvajktamasvajk
committed
Change desktop dotnet assembly lookup to fall back to nuget reference assemblies
1 parent 04f0fb0 commit 8f0f696

File tree

6 files changed

+336
-299
lines changed

6 files changed

+336
-299
lines changed

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs‎

Lines changed: 58 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
6161
try
6262
{
6363
this.dotnet = DotNet.Make(options, logger, tempWorkingDirectory);
64-
runtimeLazy = new Lazy<Runtime>(() => new Runtime(dotnet));
64+
runtimeLazy = new Lazy<Runtime>(() => new Runtime(dotnet, logger));
6565
}
6666
catch
6767
{
@@ -303,7 +303,7 @@ private void AddNetFrameworkDlls(ISet<string> dllPaths, ISet<string> frameworkLo
303303
var packagesInPrioOrder = FrameworkPackageNames.NetFrameworks;
304304

305305
var frameworkPaths = packagesInPrioOrder
306-
.Select((s, index) => (Index: index, Path: GetPackageDirectory(s)))
306+
.Select((s, index) => (Index: index, Path: GetPackageDirectory(s, packageDirectory)))
307307
.Where(pair => pair.Path is not null)
308308
.ToArray();
309309

@@ -335,6 +335,16 @@ private void AddNetFrameworkDlls(ISet<string> dllPaths, ISet<string> frameworkLo
335335
else if (fileContent.IsLegacyProjectStructureUsed)
336336
{
337337
runtimeLocation = Runtime.DesktopRuntime;
338+
339+
if (runtimeLocation is null)
340+
{
341+
logger.LogInfo("No .NET Desktop Runtime location found. Attempting to restore the .NET Framework reference assemblies manually.");
342+
343+
if (TryRestorePackageManually(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies, null))
344+
{
345+
runtimeLocation = GetPackageDirectory(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies, missingPackageDirectory);
346+
}
347+
}
338348
}
339349

340350
runtimeLocation ??= Runtime.ExecutingRuntime;
@@ -374,7 +384,7 @@ private void AddAspNetCoreFrameworkDlls(ISet<string> dllPaths, ISet<string> fram
374384
}
375385

376386
// First try to find ASP.NET Core assemblies in the NuGet packages
377-
if (GetPackageDirectory(FrameworkPackageNames.AspNetCoreFramework) is string aspNetCorePackage)
387+
if (GetPackageDirectory(FrameworkPackageNames.AspNetCoreFramework, packageDirectory) is string aspNetCorePackage)
378388
{
379389
SelectNewestFrameworkPath(aspNetCorePackage, "ASP.NET Core", dllPaths, frameworkLocations);
380390
return;
@@ -390,15 +400,15 @@ private void AddAspNetCoreFrameworkDlls(ISet<string> dllPaths, ISet<string> fram
390400

391401
private void AddMicrosoftWindowsDesktopDlls(ISet<string> dllPaths, ISet<string> frameworkLocations)
392402
{
393-
if (GetPackageDirectory(FrameworkPackageNames.WindowsDesktopFramework) is string windowsDesktopApp)
403+
if (GetPackageDirectory(FrameworkPackageNames.WindowsDesktopFramework, packageDirectory) is string windowsDesktopApp)
394404
{
395405
SelectNewestFrameworkPath(windowsDesktopApp, "Windows Desktop App", dllPaths, frameworkLocations);
396406
}
397407
}
398408

399-
private string? GetPackageDirectory(string packagePrefix)
409+
private string? GetPackageDirectory(string packagePrefix, TemporaryDirectory root)
400410
{
401-
return new DirectoryInfo(packageDirectory.DirInfo.FullName)
411+
return new DirectoryInfo(root.DirInfo.FullName)
402412
.EnumerateDirectories(packagePrefix + "*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
403413
.FirstOrDefault()?
404414
.FullName;
@@ -434,19 +444,19 @@ private void GenerateSourceFileFromImplicitUsings()
434444
}
435445

436446
// Hardcoded values from https://learn.microsoft.com/en-us/dotnet/core/project-sdk/overview#implicit-using-directives
437-
usings.UnionWith(new[] { "System", "System.Collections.Generic", "System.IO", "System.Linq", "System.Net.Http", "System.Threading",
438-
"System.Threading.Tasks" });
447+
usings.UnionWith([ "System", "System.Collections.Generic", "System.IO", "System.Linq", "System.Net.Http", "System.Threading",
448+
"System.Threading.Tasks" ]);
439449

440450
if (fileContent.UseAspNetCoreDlls)
441451
{
442-
usings.UnionWith(new[] { "System.Net.Http.Json", "Microsoft.AspNetCore.Builder", "Microsoft.AspNetCore.Hosting",
452+
usings.UnionWith([ "System.Net.Http.Json", "Microsoft.AspNetCore.Builder", "Microsoft.AspNetCore.Hosting",
443453
"Microsoft.AspNetCore.Http", "Microsoft.AspNetCore.Routing", "Microsoft.Extensions.Configuration",
444-
"Microsoft.Extensions.DependencyInjection", "Microsoft.Extensions.Hosting", "Microsoft.Extensions.Logging" });
454+
"Microsoft.Extensions.DependencyInjection", "Microsoft.Extensions.Hosting", "Microsoft.Extensions.Logging" ]);
445455
}
446456

447457
if (fileContent.UseWindowsForms)
448458
{
449-
usings.UnionWith(new[] { "System.Drawing", "System.Windows.Forms" });
459+
usings.UnionWith(["System.Drawing", "System.Windows.Forms"]);
450460
}
451461

452462
usings.UnionWith(fileContent.CustomImplicitUsings);
@@ -828,47 +838,58 @@ private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPa
828838

829839
Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, package =>
830840
{
831-
logger.LogInfo($"Restoring package {package}...");
832-
using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir"));
833-
var success = dotnet.New(tempDir.DirInfo.FullName);
841+
var success = TryRestorePackageManually(package, nugetConfig);
834842
if (!success)
835843
{
836844
return;
837845
}
838846

839-
success = dotnet.AddPackage(tempDir.DirInfo.FullName, package);
840-
if (!success)
847+
lock (sync)
841848
{
842-
return;
849+
successCount++;
843850
}
851+
});
844852

845-
var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig));
846-
if (!res.Success)
847-
{
848-
if (res.HasNugetPackageSourceError)
849-
{
850-
// Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
851-
res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true));
852-
}
853+
CompilationInfos.Add(("Successfully ran fallback nuget restore", successCount.ToString()));
853854

854-
// TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
855+
dllPaths.Add(missingPackageDirectory.DirInfo.FullName);
856+
}
855857

856-
if (!res.Success)
857-
{
858-
logger.LogInfo($"Failed to restore nuget package {package}");
859-
return;
860-
}
861-
}
858+
private bool TryRestorePackageManually(string package, string? nugetConfig)
859+
{
860+
logger.LogInfo($"Restoring package {package}...");
861+
using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir"));
862+
var success = dotnet.New(tempDir.DirInfo.FullName);
863+
if (!success)
864+
{
865+
return false;
866+
}
862867

863-
lock (sync)
868+
success = dotnet.AddPackage(tempDir.DirInfo.FullName, package);
869+
if (!success)
870+
{
871+
return false;
872+
}
873+
874+
var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig));
875+
if (!res.Success)
876+
{
877+
if (res.HasNugetPackageSourceError && nugetConfig is not null)
864878
{
865-
successCount++;
879+
// Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
880+
res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true));
866881
}
867-
});
868882

869-
CompilationInfos.Add(("Successfully ran fallback nuget restore", successCount.ToString()));
883+
// TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
870884

871-
dllPaths.Add(missingPackageDirectory.DirInfo.FullName);
885+
if (!res.Success)
886+
{
887+
logger.LogInfo($"Failed to restore nuget package {package}");
888+
return false;
889+
}
890+
}
891+
892+
return true;
872893
}
873894

874895
public void Dispose(TemporaryDirectory? dir, string name)

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FrameworkPackageNames.cs‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,25 @@
11
using System.Collections.Generic;
2-
using System.Linq;
32

43
namespace Semmle.Extraction.CSharp.DependencyFetching
54
{
65
internal static class FrameworkPackageNames
76
{
7+
public static string LatestNetFrameworkReferenceAssemblies { get; } = "microsoft.netframework.referenceassemblies.net481";
8+
89
public static string AspNetCoreFramework { get; } = "microsoft.aspnetcore.app.ref";
910

1011
public static string WindowsDesktopFramework { get; } = "microsoft.windowsdesktop.app.ref";
1112

1213
// The order of the packages is important.
13-
public static string[] NetFrameworks { get; } = new string[]
14-
{
14+
public static string[] NetFrameworks { get; } =
15+
[
1516
"microsoft.netcore.app.ref", // net7.0, ... net5.0, netcoreapp3.1, netcoreapp3.0
1617
"microsoft.netframework.referenceassemblies.", // net48, ..., net20
1718
"netstandard.library.ref", // netstandard2.1
1819
"netstandard.library" // netstandard2.0
19-
};
20+
];
2021

2122
public static IEnumerable<string> AllFrameworks { get; } =
22-
NetFrameworks
23-
.Union(new string[] { AspNetCoreFramework, WindowsDesktopFramework });
23+
[.. NetFrameworks, AspNetCoreFramework, WindowsDesktopFramework];
2424
}
2525
}

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Runtime.cs‎

Lines changed: 31 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
using System.Runtime.InteropServices;
66
using System.Text.RegularExpressions;
77
using Semmle.Util;
8+
using Semmle.Util.Logging;
89

910
namespace Semmle.Extraction.CSharp.DependencyFetching
1011
{
@@ -17,12 +18,14 @@ internal partial class Runtime
1718
private const string aspNetCoreApp = "Microsoft.AspNetCore.App";
1819

1920
private readonly IDotNet dotNet;
21+
private readonly ILogger logger;
2022
private readonly Lazy<Dictionary<string, DotNetVersion>> newestRuntimes;
2123
private Dictionary<string, DotNetVersion> NewestRuntimes => newestRuntimes.Value;
2224

23-
public Runtime(IDotNet dotNet)
25+
public Runtime(IDotNet dotNet, ILogger logger)
2426
{
2527
this.dotNet = dotNet;
28+
this.logger = logger;
2629
this.newestRuntimes = new(GetNewestRuntimes);
2730
}
2831

@@ -65,7 +68,7 @@ internal Dictionary<string, DotNetVersion> GetNewestRuntimes()
6568
/// Locates .NET Desktop Runtimes.
6669
/// This includes Mono and Microsoft.NET.
6770
/// </summary>
68-
private static IEnumerable<string> DesktopRuntimes
71+
private IEnumerable<string> DesktopRuntimes
6972
{
7073
get
7174
{
@@ -75,21 +78,38 @@ private static IEnumerable<string> DesktopRuntimes
7578
.OrderByDescending(Path.GetFileName);
7679
}
7780

78-
var monoPath = FileUtils.FindProgramOnPath(Win32.IsWindows() ? "mono.exe" : "mono");
79-
var monoDirs = monoPath is not null
80-
? new[] { Path.GetFullPath(Path.Combine(monoPath, "..", "lib", "mono")), monoPath }
81-
: new[] { "/usr/lib/mono", "/usr/local/mono", "/usr/local/bin/mono", @"C:\Program Files\Mono\lib\mono" };
81+
string? monoDir = null;
8282

83-
var dir = monoDirs.FirstOrDefault(Directory.Exists);
83+
var monoPathEnv = Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_BUILDLESS_MONO_PATH");
84+
if (monoPathEnv is not null)
85+
{
86+
if (Directory.Exists(monoPathEnv))
87+
{
88+
monoDir = monoPathEnv;
89+
}
90+
else
91+
{
92+
logger.LogError($"The directory specified in CODEQL_EXTRACTOR_CSHARP_BUILDLESS_MONO_PATH does not exist: {monoPathEnv}");
93+
}
94+
}
95+
else
96+
{
97+
var monoPath = FileUtils.FindProgramOnPath(Win32.IsWindows() ? "mono.exe" : "mono");
98+
string[] monoDirs = monoPath is not null
99+
? [Path.GetFullPath(Path.Combine(monoPath, "..", "lib", "mono")), monoPath]
100+
: ["/usr/lib/mono", "/usr/local/mono", "/usr/local/bin/mono", @"C:\Program Files\Mono\lib\mono"];
101+
102+
monoDir = monoDirs.FirstOrDefault(Directory.Exists);
103+
}
84104

85-
if (dir is not null)
105+
if (monoDir is not null)
86106
{
87-
return Directory.EnumerateDirectories(dir)
88-
.Where(d => Char.IsDigit(Path.GetFileName(d)[0]))
107+
return Directory.EnumerateDirectories(monoDir)
108+
.Where(d => char.IsDigit(Path.GetFileName(d)[0]))
89109
.OrderByDescending(Path.GetFileName);
90110
}
91111

92-
return Enumerable.Empty<string>();
112+
return [];
93113
}
94114
}
95115

‎csharp/extractor/Semmle.Extraction.Tests/Runtime.cs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ public void TestRuntime1()
4747
"Microsoft.NETCore.App 7.0.2 [/path/dotnet/shared/Microsoft.NETCore.App]"
4848
};
4949
var dotnet = new DotNetStub(listedRuntimes, null!);
50-
var runtime = new Runtime(dotnet);
50+
var runtime = new Runtime(dotnet, new LoggerStub());
5151

5252
// Execute
5353
var runtimes = runtime.GetNewestRuntimes();
@@ -73,7 +73,7 @@ public void TestRuntime2()
7373
"Microsoft.NETCore.App 8.0.0-preview.5.23280.8 [/path/dotnet/shared/Microsoft.NETCore.App]"
7474
};
7575
var dotnet = new DotNetStub(listedRuntimes, null!);
76-
var runtime = new Runtime(dotnet);
76+
var runtime = new Runtime(dotnet, new LoggerStub());
7777

7878
// Execute
7979
var runtimes = runtime.GetNewestRuntimes();
@@ -96,7 +96,7 @@ public void TestRuntime3()
9696
"Microsoft.NETCore.App 8.0.0-preview.5.23280.8 [/path/dotnet/shared/Microsoft.NETCore.App]"
9797
};
9898
var dotnet = new DotNetStub(listedRuntimes, null!);
99-
var runtime = new Runtime(dotnet);
99+
var runtime = new Runtime(dotnet, new LoggerStub());
100100

101101
// Execute
102102
var runtimes = runtime.GetNewestRuntimes();
@@ -125,7 +125,7 @@ public void TestRuntime4()
125125
@"Microsoft.WindowsDesktop.App 7.0.4 [C:\Program Files\dotnet\shared\Microsoft.WindowsDesktop.App]"
126126
};
127127
var dotnet = new DotNetStub(listedRuntimes, null!);
128-
var runtime = new Runtime(dotnet);
128+
var runtime = new Runtime(dotnet, new LoggerStub());
129129

130130
// Execute
131131
var runtimes = runtime.GetNewestRuntimes();

0 commit comments

Comments
 (0)