From 2054c869fb53626cfe801c01460226eace1c27cc Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Tue, 18 Mar 2025 09:51:33 +0100 Subject: [PATCH 1/2] C#: Fix buildless fallback restore logic When dotnet core projects are restored, the dependency manager precisely tracks the referenced package folders. The fallback restore logic ignored the precise usage list and instead considered all subfolders in the restore location to be referenced, even though not all subfolders were added to the dependency list. This meant that packages downloaded in partially successful restores were available on disk, but not added to the dependency list by the normal restore process, and skipped by the fallback restore process. This commit fixes this problem by ensuring that the fallback restore logic doesn't consider all subfolders in the restore location to be referenced, but only those that were added to the dependency list by the normal restore process. --- .../NugetPackageRestorer.cs | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs index 393e37579b71..dabc95b0d596 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs @@ -109,7 +109,7 @@ public HashSet Restore() if (checkNugetFeedResponsiveness && !CheckFeeds(out explicitFeeds)) { // todo: we could also check the reachability of the inherited nuget feeds, but to use those in the fallback we would need to handle authentication too. - var unresponsiveMissingPackageLocation = DownloadMissingPackagesFromSpecificFeeds(explicitFeeds); + var unresponsiveMissingPackageLocation = DownloadMissingPackagesFromSpecificFeeds([], explicitFeeds); return unresponsiveMissingPackageLocation is null ? [] : [unresponsiveMissingPackageLocation]; @@ -166,11 +166,11 @@ public HashSet Restore() .ToList(); assemblyLookupLocations.UnionWith(paths.Select(p => new AssemblyLookupLocation(p))); - LogAllUnusedPackages(dependencies); + var usedPackageNames = GetAllUsedPackageDirNames(dependencies); var missingPackageLocation = checkNugetFeedResponsiveness - ? DownloadMissingPackagesFromSpecificFeeds(explicitFeeds) - : DownloadMissingPackages(); + ? DownloadMissingPackagesFromSpecificFeeds(usedPackageNames, explicitFeeds) + : DownloadMissingPackages(usedPackageNames); if (missingPackageLocation is not null) { @@ -297,21 +297,21 @@ private void RestoreProjects(IEnumerable projects, out ConcurrentBag? feedsFromNugetConfigs) + private AssemblyLookupLocation? DownloadMissingPackagesFromSpecificFeeds(IEnumerable usedPackageNames, HashSet? feedsFromNugetConfigs) { var reachableFallbackFeeds = GetReachableFallbackNugetFeeds(feedsFromNugetConfigs); if (reachableFallbackFeeds.Count > 0) { - return DownloadMissingPackages(fallbackNugetFeeds: reachableFallbackFeeds); + return DownloadMissingPackages(usedPackageNames, fallbackNugetFeeds: reachableFallbackFeeds); } logger.LogWarning("Skipping download of missing packages from specific feeds as no fallback Nuget feeds are reachable."); return null; } - private AssemblyLookupLocation? DownloadMissingPackages(IEnumerable? fallbackNugetFeeds = null) + private AssemblyLookupLocation? DownloadMissingPackages(IEnumerable usedPackageNames, IEnumerable? fallbackNugetFeeds = null) { - var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(PackageDirectory.DirInfo); + var alreadyDownloadedPackages = usedPackageNames.Select(p => p.ToLowerInvariant()); var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames(); var notYetDownloadedPackages = new HashSet(fileContent.AllPackages); @@ -418,17 +418,23 @@ private void RestoreProjects(IEnumerable projects, out ConcurrentBag GetAllUsedPackageDirNames(DependencyContainer dependencies) { var allPackageDirectories = GetAllPackageDirectories(); logger.LogInfo($"Restored {allPackageDirectories.Count} packages"); logger.LogInfo($"Found {dependencies.Packages.Count} packages in project.assets.json files"); - allPackageDirectories - .Where(package => !dependencies.Packages.Contains(package)) + var usage = allPackageDirectories.Select(package => (package, isUsed: dependencies.Packages.Contains(package))); + + usage + .Where(package => !package.isUsed) .Order() - .ForEach(package => logger.LogDebug($"Unused package: {package}")); + .ForEach(package => logger.LogDebug($"Unused package: {package.package}")); + + return usage + .Where(package => package.isUsed) + .Select(package => package.package); } private ICollection GetAllPackageDirectories() From e20c46a14a776cdf8b1ecdd952cf4908949438f7 Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Fri, 21 Mar 2025 08:24:22 +0100 Subject: [PATCH 2/2] Add change note --- csharp/ql/src/change-notes/2025-03-21-dependency-fetching.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 csharp/ql/src/change-notes/2025-03-21-dependency-fetching.md diff --git a/csharp/ql/src/change-notes/2025-03-21-dependency-fetching.md b/csharp/ql/src/change-notes/2025-03-21-dependency-fetching.md new file mode 100644 index 000000000000..84c6a9721dc8 --- /dev/null +++ b/csharp/ql/src/change-notes/2025-03-21-dependency-fetching.md @@ -0,0 +1,4 @@ +--- +category: minorAnalysis +--- +* Improved dependency resolution in `build-mode: none` extraction to handle failing `dotnet restore` processes that managed to download a subset of the dependencies before the failure.